From 947feaa0c936c4327ec441400f607f41d4dd31f7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Mar 2026 00:42:46 +0000 Subject: [PATCH 001/258] Add gpt-5.4 and gpt-5.4-2026-03-05 model support https://claude.ai/code/session_01HwqZ4WeDCrspfF8E7STiPA --- llm/default_plugins/openai_models.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index ccca240cf..1ff2cfcb0 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -233,6 +233,25 @@ def register_models(register): ) # "gpt-5.2-pro" is Responses API only + # GPT-5.4 + for model_id in ("gpt-5.4", "gpt-5.4-2026-03-05"): + register( + Chat( + model_id, + vision=True, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + AsyncChat( + model_id, + vision=True, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + ) + # The -instruct completion model register( Completion("gpt-3.5-turbo-instruct", default_max_tokens=256), From 683ca204b22623da922fb574a73eabe0aff5630b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 17 Mar 2026 11:22:34 -0700 Subject: [PATCH 002/258] Ensure -x/--xl work with -t --- llm/cli.py | 5 +++-- tests/test_templates.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index fc6fb41b1..e2384dbcb 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -677,8 +677,9 @@ def read_prompt(): template_obj = load_template(template) except LoadTemplateError as ex: raise click.ClickException(str(ex)) - extract = template_obj.extract - extract_last = template_obj.extract_last + if not (extract or extract_last): + extract = template_obj.extract + extract_last = template_obj.extract_last # Combine with template fragments/system_fragments if template_obj.fragments: fragments = [*template_obj.fragments, *fragments] diff --git a/tests/test_templates.py b/tests/test_templates.py index dcdabd9bc..d63187d40 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -413,6 +413,21 @@ def test_execute_prompt_from_template_path(): } +def test_template_respects_cli_extract_flag( + mocked_openai_chat_returning_fenced_code, templates_path +): + (templates_path / "code.yaml").write_text("prompt: Write code", "utf-8") + runner = CliRunner() + result = runner.invoke( + cli, + ["-t", "code", "-m", "gpt-4o-mini", "--key", "x", "-x"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert "```" not in result.output + assert result.output.strip() == "function foo() {\n return 'bar';\n}" + + FUNCTIONS_EXAMPLE = """ def greet(name: str) -> str: return f"Hello, {name}!" From 07dccc00eff5feaa3de2d6ded45c62438e2798b6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 17 Mar 2026 11:23:28 -0700 Subject: [PATCH 003/258] GPT-5.4, 5.4-mini, 5.4-nano Closes #1376 --- docs/openai-models.md | 6 ++ docs/usage.md | 132 +++++++++++++++++++++++++++ llm/default_plugins/openai_models.py | 11 ++- 3 files changed, 148 insertions(+), 1 deletion(-) diff --git a/docs/openai-models.md b/docs/openai-models.md index ce064c291..ce3365dc4 100644 --- a/docs/openai-models.md +++ b/docs/openai-models.md @@ -69,6 +69,12 @@ OpenAI Chat: gpt-5.1 OpenAI Chat: gpt-5.1-chat-latest OpenAI Chat: gpt-5.2 OpenAI Chat: gpt-5.2-chat-latest +OpenAI Chat: gpt-5.4 +OpenAI Chat: gpt-5.4-2026-03-05 +OpenAI Chat: gpt-5.4-mini +OpenAI Chat: gpt-5.4-mini-2026-03-17 +OpenAI Chat: gpt-5.4-nano +OpenAI Chat: gpt-5.4-nano-2026-03-17 OpenAI Completion: gpt-3.5-turbo-instruct (aliases: 3.5-instruct, chatgpt-instruct) ``` diff --git a/docs/usage.md b/docs/usage.md index 5298e85e2..651928907 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1295,6 +1295,138 @@ OpenAI Chat: gpt-5.2-chat-latest Keys: key: openai env_var: OPENAI_API_KEY +OpenAI Chat: gpt-5.4 + Options: + temperature: float + max_tokens: int + top_p: float + frequency_penalty: float + presence_penalty: float + stop: str + logit_bias: dict, str + seed: int + json_object: boolean + reasoning_effort: str + Attachment types: + application/pdf, image/gif, image/jpeg, image/png, image/webp + Features: + - streaming + - schemas + - tools + - async + Keys: + key: openai + env_var: OPENAI_API_KEY +OpenAI Chat: gpt-5.4-2026-03-05 + Options: + temperature: float + max_tokens: int + top_p: float + frequency_penalty: float + presence_penalty: float + stop: str + logit_bias: dict, str + seed: int + json_object: boolean + reasoning_effort: str + Attachment types: + application/pdf, image/gif, image/jpeg, image/png, image/webp + Features: + - streaming + - schemas + - tools + - async + Keys: + key: openai + env_var: OPENAI_API_KEY +OpenAI Chat: gpt-5.4-mini + Options: + temperature: float + max_tokens: int + top_p: float + frequency_penalty: float + presence_penalty: float + stop: str + logit_bias: dict, str + seed: int + json_object: boolean + reasoning_effort: str + Attachment types: + application/pdf, image/gif, image/jpeg, image/png, image/webp + Features: + - streaming + - schemas + - tools + - async + Keys: + key: openai + env_var: OPENAI_API_KEY +OpenAI Chat: gpt-5.4-mini-2026-03-17 + Options: + temperature: float + max_tokens: int + top_p: float + frequency_penalty: float + presence_penalty: float + stop: str + logit_bias: dict, str + seed: int + json_object: boolean + reasoning_effort: str + Attachment types: + application/pdf, image/gif, image/jpeg, image/png, image/webp + Features: + - streaming + - schemas + - tools + - async + Keys: + key: openai + env_var: OPENAI_API_KEY +OpenAI Chat: gpt-5.4-nano + Options: + temperature: float + max_tokens: int + top_p: float + frequency_penalty: float + presence_penalty: float + stop: str + logit_bias: dict, str + seed: int + json_object: boolean + reasoning_effort: str + Attachment types: + application/pdf, image/gif, image/jpeg, image/png, image/webp + Features: + - streaming + - schemas + - tools + - async + Keys: + key: openai + env_var: OPENAI_API_KEY +OpenAI Chat: gpt-5.4-nano-2026-03-17 + Options: + temperature: float + max_tokens: int + top_p: float + frequency_penalty: float + presence_penalty: float + stop: str + logit_bias: dict, str + seed: int + json_object: boolean + reasoning_effort: str + Attachment types: + application/pdf, image/gif, image/jpeg, image/png, image/webp + Features: + - streaming + - schemas + - tools + - async + Keys: + key: openai + env_var: OPENAI_API_KEY OpenAI Completion: gpt-3.5-turbo-instruct (aliases: 3.5-instruct, chatgpt-instruct) Options: temperature: float diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 1ff2cfcb0..849d5fde2 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -234,7 +234,14 @@ def register_models(register): # "gpt-5.2-pro" is Responses API only # GPT-5.4 - for model_id in ("gpt-5.4", "gpt-5.4-2026-03-05"): + for model_id in ( + "gpt-5.4", + "gpt-5.4-2026-03-05", + "gpt-5.4-mini", + "gpt-5.4-mini-2026-03-17", + "gpt-5.4-nano", + "gpt-5.4-nano-2026-03-17", + ): register( Chat( model_id, @@ -502,10 +509,12 @@ def validate_logit_bias(cls, logit_bias): class ReasoningEffortEnum(str, Enum): + none = "none" minimal = "minimal" low = "low" medium = "medium" high = "high" + xhigh = "xhigh" class OptionsForReasoning(SharedOptions): From c8889e0a76abf6d44a4473525b460d8c614ae441 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 17 Mar 2026 11:25:12 -0700 Subject: [PATCH 004/258] Ran black --- llm/cli.py | 32 ++++++++++---------------------- llm/embeddings.py | 4 +--- llm/embeddings_migrations.py | 12 ++++-------- llm/utils.py | 1 - tests/test_chat.py | 6 ++---- tests/test_fragments_cli.py | 8 ++------ tests/test_llm.py | 4 ++++ tests/test_llm_logs.py | 7 ++----- tests/test_migrate.py | 1 - tests/test_plugins.py | 6 ++---- tests/test_templates.py | 6 ++---- tests/test_tools.py | 1 - tests/test_tools_streaming.py | 1 - 13 files changed, 29 insertions(+), 60 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index e2384dbcb..c78c0be6a 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -1722,16 +1722,14 @@ def logs_list( if any_tools: # Any response that involved at least one tool result - where_bits.append( - """ + where_bits.append(""" exists ( select 1 from tool_results where tool_results.response_id = responses.id ) - """ - ) + """) if tools: tools_by_name = get_tools() # Filter responses by tools (must have ALL of the named tools, including plugin) @@ -1742,8 +1740,7 @@ def logs_list( except KeyError: raise click.ClickException(f"Unknown tool: {tool_name}") - tool_clauses.append( - f""" + tool_clauses.append(f""" exists ( select 1 from tool_results @@ -1752,8 +1749,7 @@ def logs_list( and tools.name = :tool{i} and tools.plugin = :plugin{i} ) - """ - ) + """) sql_params[f"tool{i}"] = tool_name sql_params[f"plugin{i}"] = plugin_name @@ -2483,9 +2479,7 @@ def schemas_list(path, database, queries, full, json_, nl): on responses.schema_id = schemas.id {} group by responses.schema_id order by recently_used - """.format( - where_sql - ) + """.format(where_sql) rows = db.query(sql, params) if json_ or nl: @@ -2824,13 +2818,11 @@ def fragments_list(queries, aliases, json_): param_count += 1 p = f"p{param_count}" params[p] = q - where_bits.append( - f""" + where_bits.append(f""" (fragments.hash = :{p} or fragment_aliases.alias = :{p} or fragments.source like '%' || :{p} || '%' or fragments.content like '%' || :{p} || '%') - """ - ) + """) where = "\n and\n ".join(where_bits) if where: where = " where " + where @@ -2852,9 +2844,7 @@ def fragments_list(queries, aliases, json_): group by fragments.id, fragments.hash, fragments.content, fragments.datetime_utc, fragments.source order by fragments.datetime_utc - """.format( - where=where - ) + """.format(where=where) results = list(db.query(sql, params)) for result in results: result["aliases"] = json.loads(result["aliases"]) @@ -3544,8 +3534,7 @@ def embed_db_collections(database, json_): db = sqlite_utils.Database(str(database)) if not db["collections"].exists(): raise click.ClickException("No collections table found in {}".format(database)) - rows = db.query( - """ + rows = db.query(""" select collections.name, collections.model, @@ -3555,8 +3544,7 @@ def embed_db_collections(database, json_): on collections.id = embeddings.collection_id group by collections.name, collections.model - """ - ) + """) if json_: click.echo(json.dumps(list(rows), indent=4)) else: diff --git a/llm/embeddings.py b/llm/embeddings.py index 5c9bf8ffa..90b983a11 100644 --- a/llm/embeddings.py +++ b/llm/embeddings.py @@ -202,9 +202,7 @@ def embed_multi_with_metadata( """ select id from embeddings where collection_id = ? and content_hash in ({}) - """.format( - ",".join("?" for _ in items_and_hashes) - ), + """.format(",".join("?" for _ in items_and_hashes)), [collection_id] + [item_and_hash[1] for item_and_hash in items_and_hashes], ) diff --git a/llm/embeddings_migrations.py b/llm/embeddings_migrations.py index 600ad204d..69545f3ea 100644 --- a/llm/embeddings_migrations.py +++ b/llm/embeddings_migrations.py @@ -63,20 +63,16 @@ def random_md5(): db.conn.create_function("temp_random_md5", 0, random_md5) with db.conn: - db.execute( - """ + db.execute(""" update embeddings set content_hash = temp_md5(content) where content is not null - """ - ) - db.execute( - """ + """) + db.execute(""" update embeddings set content_hash = temp_random_md5() where content is null - """ - ) + """) db["embeddings"].create_index(["content_hash"]) diff --git a/llm/utils.py b/llm/utils.py index 58194bd6a..587f19284 100644 --- a/llm/utils.py +++ b/llm/utils.py @@ -16,7 +16,6 @@ from ulid import ULID - MIME_TYPE_FIXES = { "audio/wave": "audio/wav", } diff --git a/tests/test_chat.py b/tests/test_chat.py index a0d010c2a..4563f301f 100644 --- a/tests/test_chat.py +++ b/tests/test_chat.py @@ -308,13 +308,11 @@ def test_llm_chat_creates_log_database(tmpdir, monkeypatch, custom_database_path @pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows") def test_chat_tools(logs_db): runner = CliRunner() - functions = textwrap.dedent( - """ + functions = textwrap.dedent(""" def upper(text: str) -> str: "Convert text to upper case" return text.upper() - """ - ) + """) result = runner.invoke( llm.cli.cli, ["chat", "-m", "echo", "--functions", functions], diff --git a/tests/test_fragments_cli.py b/tests/test_fragments_cli.py index 8606205bd..5975c9e70 100644 --- a/tests/test_fragments_cli.py +++ b/tests/test_fragments_cli.py @@ -106,9 +106,7 @@ def test_fragments_list(user_path): ) result = runner.invoke(cli, ["fragments", "list"]) assert result.exit_code == 0 - assert result.output.strip() == ( - textwrap.dedent( - """ + assert result.output.strip() == (textwrap.dedent(""" - hash: hash2 aliases: [] datetime_utc: '2022-10-01T00:00:00Z' @@ -125,9 +123,7 @@ def test_fragments_list(user_path): datetime_utc: '2024-10-01T00:00:00Z' source: file3.txt content: '3' - """ - ).strip() - ) + """).strip()) @mock.patch.dict(os.environ, {"OPENAI_API_KEY": "X"}) diff --git a/tests/test_llm.py b/tests/test_llm.py index 334d40f6f..8b307cc2a 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -568,6 +568,8 @@ def test_get_models(): assert all(isinstance(model, (llm.Model, llm.KeyModel)) for model in models) model_ids = [model.model_id for model in models] assert "gpt-4o-mini" in model_ids + assert "gpt-5.4-mini" in model_ids + assert "gpt-5.4-nano" in model_ids # Ensure no model_ids are duplicated # https://github.com/simonw/llm/issues/667 assert len(model_ids) == len(set(model_ids)) @@ -580,6 +582,8 @@ def test_get_async_models(): ) model_ids = [model.model_id for model in models] assert "gpt-4o-mini" in model_ids + assert "gpt-5.4-mini" in model_ids + assert "gpt-5.4-nano" in model_ids def test_mock_model(mock_model): diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index 79f705521..0af16b269 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -15,7 +15,6 @@ from ulid import ULID import yaml - SINGLE_ID = "5843577700ba729bb14c327b30441885" MULTI_ID = "4860edd987df587d042a9eb2b299ce5c" @@ -935,12 +934,10 @@ def test_expand_fragment_markdown(fragments_fixture): def test_logs_tools(logs_db): runner = CliRunner() - code = textwrap.dedent( - """ + code = textwrap.dedent(""" def demo(): return "one\\ntwo\\nthree" - """ - ) + """) result1 = runner.invoke( cli, [ diff --git a/tests/test_migrate.py b/tests/test_migrate.py index e7f70bc3e..c526117eb 100644 --- a/tests/test_migrate.py +++ b/tests/test_migrate.py @@ -4,7 +4,6 @@ import pytest import sqlite_utils - EXPECTED = { "id": str, "model": str, diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 52203925f..6777fd585 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -174,8 +174,7 @@ def register_fragment_loaders(self, register): cli.cli, ["-m", "echo", "-f", "mixed:x"], catch_exceptions=False ) assert result3.exit_code == 0 - result3.output.strip == textwrap.dedent( - """\ + result3.output.strip == textwrap.dedent("""\ system: @@ -184,8 +183,7 @@ def register_fragment_loaders(self, register): attachments: - https://example.com/attachment.png - """ - ).strip() + """).strip() finally: plugins.pm.unregister(name="FragmentLoadersPlugin") diff --git a/tests/test_templates.py b/tests/test_templates.py index d63187d40..38229619b 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -464,8 +464,7 @@ def register_tools(self, register): def test_tools_in_templates( source, expected_tool_success, expected_functions_success, httpx_mock, tmpdir ): - template_yaml = textwrap.dedent( - """ + template_yaml = textwrap.dedent(""" name: test tools: - llm_version @@ -473,8 +472,7 @@ def test_tools_in_templates( functions: | def demo(): return "Demo" - """ - ) + """) args = [] def before(): diff --git a/tests/test_tools.py b/tests/test_tools.py index c61154f22..b849779cc 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -11,7 +11,6 @@ import sqlite_utils import time - API_KEY = os.environ.get("PYTEST_OPENAI_API_KEY", None) or "badkey" diff --git a/tests/test_tools_streaming.py b/tests/test_tools_streaming.py index 31ed2cb71..2e3967f7c 100644 --- a/tests/test_tools_streaming.py +++ b/tests/test_tools_streaming.py @@ -3,7 +3,6 @@ import os import pytest - API_KEY = os.environ.get("PYTEST_OPENAI_API_KEY", None) or "badkey" From b8cb53a2a1a21560d7bf1ea8b2c10096842c3142 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 17 Mar 2026 12:19:13 -0700 Subject: [PATCH 005/258] Release 0.29 Refs #1322, #1376 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6c3c5e70c..77e92f89b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm" -version = "0.28" +version = "0.29" description = "CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine." readme = { file = "README.md", content-type = "text/markdown" } authors = [ From c7cf7e506ebec8cabeae90dc70a9482e534be220 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 17 Mar 2026 12:23:43 -0700 Subject: [PATCH 006/258] Changelog for 0.29 --- docs/changelog.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 24bf2711a..321e7af79 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,12 @@ # Changelog +(v0_29)= +## 0.29 (2025-03-17) + +- The `-t/--template` option now works correctly with the `-x/--extract` and `--xl/--extract-last` flags. +- `llm logs` now shows any additional model options in the Markdown output. [#1322](https://github.com/simonw/llm/issues/1322) +- New OpenAI models: `gpt-5.4`, `gpt-5.4-mini`, `gpt-5.4-nano`. [#1376](https://github.com/simonw/llm/issues/1376) + (v0_28)= ## 0.28 (2025-12-12) From 40f7b8f2fa7900b5af4fbbbd7051913afe39c537 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 31 Mar 2026 11:53:15 -0700 Subject: [PATCH 007/258] Ran cog --- docs/fragments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/fragments.md b/docs/fragments.md index 572f84049..c623545b7 100644 --- a/docs/fragments.md +++ b/docs/fragments.md @@ -30,7 +30,7 @@ from importlib.metadata import version llm_version = version("llm") cog.out(f'The URL will be fetched with the user-agent `llm/{llm_version} (https://llm.datasette.io/)`.') ]]]--> -The URL will be fetched with the user-agent `llm/0.28 (https://llm.datasette.io/)`. +The URL will be fetched with the user-agent `llm/0.29 (https://llm.datasette.io/)`. The `-f` option can be used multiple times to combine together multiple fragments. From 49d4e54639bf55039d12a16be6e44f5eeb1d3b15 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 31 Mar 2026 12:45:38 -0700 Subject: [PATCH 008/258] register_models() model_aliases parameter --- README.md | 2 +- docs/plugins/plugin-hooks.md | 9 ++++++++- llm/__init__.py | 2 +- llm/hookspecs.py | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 92cde72d1..b453b33a4 100644 --- a/README.md +++ b/README.md @@ -254,7 +254,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Just for fun](https://llm.datasette.io/en/stable/plugins/directory.html#just-for-fun) * [Plugin hooks](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html) * [register_commands(cli)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-commands-cli) - * [register_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-models-register) + * [register_models(register, model_aliases)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-models-register-model-aliases) * [register_embedding_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-embedding-models-register) * [register_tools(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-tools-register) * [register_template_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-template-loaders-register) diff --git a/docs/plugins/plugin-hooks.md b/docs/plugins/plugin-hooks.md index 062919019..7adc5fe01 100644 --- a/docs/plugins/plugin-hooks.md +++ b/docs/plugins/plugin-hooks.md @@ -28,7 +28,7 @@ def register_commands(cli): This new command will be added to `llm --help` and can be run using `llm hello-world`. (plugin-hooks-register-models)= -## register_models(register) +## register_models(register, model_aliases) This hook can be used to register one or more additional models. @@ -60,8 +60,15 @@ def register_models(register): ``` This demonstrates how to register a model with both sync and async versions, and how to specify an alias for that model. +The `model_aliases` parameter is a list of {class}`~llm.ModelWithAliases` objects representing all models registered so far by other plugins. Plugins that use `@llm.hookimpl(trylast=True)` can use this to inspect or modify models registered by other plugins. Both parameters are optional - plugins can accept just `register`, just `model_aliases`, or both. + The {ref}`model plugin tutorial ` describes how to use this hook in detail. Asynchronous models {ref}`are described here `. +```{eval-rst} +.. autoclass:: llm.ModelWithAliases + :exclude-members: matches +``` + (plugin-hooks-register-embedding-models)= ## register_embedding_models(register) diff --git a/llm/__init__.py b/llm/__init__.py index 09ee01844..8ddd775f6 100644 --- a/llm/__init__.py +++ b/llm/__init__.py @@ -108,7 +108,7 @@ def register(model, async_model=None, aliases=None): model_aliases.append(ModelWithAliases(model, async_model, alias_list)) load_plugins() - pm.hook.register_models(register=register) + pm.hook.register_models(register=register, model_aliases=model_aliases) return model_aliases diff --git a/llm/hookspecs.py b/llm/hookspecs.py index a244b007f..7ab555199 100644 --- a/llm/hookspecs.py +++ b/llm/hookspecs.py @@ -11,7 +11,7 @@ def register_commands(cli): @hookspec -def register_models(register): +def register_models(register, model_aliases): "Register additional model instances representing LLM models that can be called" From 1562a8f444148b5136a5865d66f0fb8e5daf2a30 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 31 Mar 2026 12:54:00 -0700 Subject: [PATCH 009/258] Added some more autoclass docs, with new doctrings --- README.md | 1 + docs/plugins/advanced-model-plugins.md | 8 ++++++++ docs/python-api.md | 22 +++++++++++++++++++++- llm/__init__.py | 1 + llm/models.py | 20 ++++++++++++++++++++ 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b453b33a4..2e364bd40 100644 --- a/README.md +++ b/README.md @@ -302,6 +302,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Token usage](https://llm.datasette.io/en/stable/python-api.html#token-usage) * [Streaming responses](https://llm.datasette.io/en/stable/python-api.html#streaming-responses) * [Async models](https://llm.datasette.io/en/stable/python-api.html#async-models) + * [`AsyncResponse`](https://llm.datasette.io/en/stable/python-api.html#llm.AsyncResponse) * [Tool functions can be sync or async](https://llm.datasette.io/en/stable/python-api.html#tool-functions-can-be-sync-or-async) * [Tool use for async models](https://llm.datasette.io/en/stable/python-api.html#tool-use-for-async-models) * [Conversations](https://llm.datasette.io/en/stable/python-api.html#conversations) diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index f7b362557..d65edf629 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -105,6 +105,14 @@ def register_models(register): ) ``` +The `prompt` object passed to your `execute()` method is an instance of {class}`~llm.Prompt`: + +```{eval-rst} +.. autoclass:: llm.Prompt + :members: prompt, system + :exclude-members: model, options +``` + (advanced-model-plugins-schemas)= ## Supporting schemas diff --git a/docs/python-api.md b/docs/python-api.md index f42789fb1..f51e8b8a9 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -79,6 +79,11 @@ response = model.prompt( ``` Use `llm.Attachment(content=b"binary image content here")` to pass binary content directly. +```{eval-rst} +.. autoclass:: llm.Attachment + :members: resolve_type, content_bytes, base64_content +``` + You can check which attachment types (if any) a model supports using the `model.attachment_types` set: ```python @@ -486,7 +491,9 @@ Usage(input=5, 'tokenCount': 2}], 'promptTokensDetails': [{'modality': 'TEXT', 'tokenCount': 5}]}) ``` -The `.input` and `.output` properties are integers representing the number of input and output tokens. The `.details` property may be a dictionary with additional custom values that vary by model. +```{eval-rst} +.. autoclass:: llm.Usage +``` (python-api-streaming-responses)= @@ -503,6 +510,12 @@ The `response.text()` method described earlier does this for you - it runs throu If a response has been evaluated, `response.text()` will continue to return the same string. +```{eval-rst} +.. autoclass:: llm.Response + :members: text, json, usage, tool_calls, on_done + :exclude-members: fake, from_row, log_to_db +``` + (python-api-async)= ## Async models @@ -529,6 +542,13 @@ async for chunk in model.prompt( ): print(chunk, end="", flush=True) ``` + +```{eval-rst} +.. autoclass:: llm.AsyncResponse + :members: text, json, usage, tool_calls, on_done + :exclude-members: fake, from_row, log_to_db +``` + This `await model.prompt()` method takes the same arguments as the synchronous `model.prompt()` method, for options and attachments and `key=` and suchlike. (python-api-async-tools)= diff --git a/llm/__init__.py b/llm/__init__.py index 8ddd775f6..27248b336 100644 --- a/llm/__init__.py +++ b/llm/__init__.py @@ -24,6 +24,7 @@ ToolCall, ToolOutput, ToolResult, + Usage, ) from .utils import schema_dsl, Fragment from .embeddings import Collection diff --git a/llm/models.py b/llm/models.py index 5e7676eb1..e3c0db8f8 100644 --- a/llm/models.py +++ b/llm/models.py @@ -46,6 +46,8 @@ @dataclass class Usage: + "Token usage information from a model response." + input: Optional[int] = None output: Optional[int] = None details: Optional[Dict[str, Any]] = None @@ -53,6 +55,8 @@ class Usage: @dataclass class Attachment: + "An attachment (image, audio, etc) to include with a prompt." + type: Optional[str] = None path: Optional[str] = None url: Optional[str] = None @@ -325,6 +329,8 @@ class CancelToolCall(Exception): @dataclass class Prompt: + "The prompt being sent to the model." + _prompt: Optional[str] model: "Model" fragments: Optional[List[Union[str, Fragment]]] @@ -1001,10 +1007,13 @@ def log_to_db(self, db): class Response(_BaseResponse): + "Sync response from a model." + model: "Model" conversation: Optional["Conversation"] = None def on_done(self, callback): + "Register a callback to be called when the response is complete." if not self._done: self.done_callbacks.append(callback) else: @@ -1022,6 +1031,7 @@ def _force(self): list(self) def text(self) -> str: + "Return the full text of the response, executing the prompt if needed." self._force() return "".join(self._chunks) @@ -1127,6 +1137,7 @@ def execute_tool_calls( return tool_results def tool_calls(self) -> List[ToolCall]: + "Return the list of tool calls made during this response." self._force() return self._tool_calls @@ -1134,6 +1145,7 @@ def tool_calls_or_raise(self) -> List[ToolCall]: return self.tool_calls() def json(self) -> Optional[Dict[str, Any]]: + "Return the raw JSON response from the model, if available." self._force() return self.response_json @@ -1146,6 +1158,7 @@ def datetime_utc(self) -> str: return self._start_utcnow.isoformat() if self._start_utcnow else "" def usage(self) -> Usage: + "Return token usage information for this response." self._force() return Usage( input=self.input_tokens, @@ -1198,6 +1211,8 @@ def __repr__(self): class AsyncResponse(_BaseResponse): + "Async response from a model." + model: "AsyncModel" conversation: Optional["AsyncConversation"] = None @@ -1206,6 +1221,7 @@ def from_row(cls, db, row, _async=False): return super().from_row(db, row, _async=True) async def on_done(self, callback): + "Register a callback to be called when the response is complete." if not self._done: self.done_callbacks.append(callback) else: @@ -1443,10 +1459,12 @@ def text_or_raise(self) -> str: return "".join(self._chunks) async def text(self) -> str: + "Return the full text of the response, executing the prompt if needed." await self._force() return "".join(self._chunks) async def tool_calls(self) -> List[ToolCall]: + "Return the list of tool calls made during this response." await self._force() return self._tool_calls @@ -1456,6 +1474,7 @@ def tool_calls_or_raise(self) -> List[ToolCall]: return self._tool_calls async def json(self) -> Optional[Dict[str, Any]]: + "Return the raw JSON response from the model, if available." await self._force() return self.response_json @@ -1468,6 +1487,7 @@ async def datetime_utc(self) -> str: return self._start_utcnow.isoformat() if self._start_utcnow else "" async def usage(self) -> Usage: + "Return token usage information for this response." await self._force() return Usage( input=self.input_tokens, From 946e4336982ff1d7490557f1192f22acbbb33a6a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 31 Mar 2026 13:12:18 -0700 Subject: [PATCH 010/258] More docstrings and autoclass embeds in docs --- README.md | 1 + docs/embeddings/writing-plugins.md | 5 +++++ docs/python-api.md | 10 ++++++++++ llm/models.py | 13 +++++++++++++ llm/templates.py | 6 ++++++ 5 files changed, 35 insertions(+) diff --git a/README.md b/README.md index 2e364bd40..860446f06 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Retrieving similar items](https://llm.datasette.io/en/stable/embeddings/python-api.html#retrieving-similar-items) * [SQL schema](https://llm.datasette.io/en/stable/embeddings/python-api.html#sql-schema) * [Writing plugins to add new embedding models](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html) + * [`EmbeddingModel`](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html#llm.EmbeddingModel) * [Embedding binary content](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html#embedding-binary-content) * [Embedding storage format](https://llm.datasette.io/en/stable/embeddings/storage.html) * [Plugins](https://llm.datasette.io/en/stable/plugins/index.html) diff --git a/docs/embeddings/writing-plugins.md b/docs/embeddings/writing-plugins.md index 0e5cd0f7d..ec8ff8ae8 100644 --- a/docs/embeddings/writing-plugins.md +++ b/docs/embeddings/writing-plugins.md @@ -14,6 +14,11 @@ There are two components to an embedding model plugin: The following example uses the [sentence-transformers](https://github.com/UKPLab/sentence-transformers) package to provide access to the [MiniLM-L6](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) embedding model. +```{eval-rst} +.. autoclass:: llm.EmbeddingModel + :members: embed, embed_multi, embed_batch +``` + ```python import llm from sentence_transformers import SentenceTransformer diff --git a/docs/python-api.md b/docs/python-api.md index f51e8b8a9..202d88dc1 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -222,6 +222,10 @@ def generate_image(prompt: str) -> llm.ToolOutput: ) ``` +```{eval-rst} +.. autoclass:: llm.ToolOutput +``` + (python-api-toolbox)= #### Toolbox classes @@ -263,6 +267,12 @@ class Memory(llm.Toolbox): "Return a list of keys" return list(self._get_memory().keys()) ``` + +```{eval-rst} +.. autoclass:: llm.Toolbox + :members: tools, add_tool, prepare, prepare_async +``` + You can then use that from Python like this: ```python model = llm.get_model("gpt-4.1-mini") diff --git a/llm/models.py b/llm/models.py index e3c0db8f8..892a4f2ba 100644 --- a/llm/models.py +++ b/llm/models.py @@ -77,6 +77,7 @@ def id(self): return self._id def resolve_type(self): + "Return the content type, guessing from content if not specified." if self.type: return self.type # Derive it from path or url or content @@ -91,6 +92,7 @@ def resolve_type(self): raise ValueError("Attachment has no type and no content to derive it from") def content_bytes(self): + "Return the binary content, reading from path or URL if needed." content = self.content if not content: if self.path: @@ -102,6 +104,7 @@ def content_bytes(self): return content def base64_content(self): + "Return the content as a base64-encoded string." return base64.b64encode(self.content_bytes()).decode("utf-8") def __repr__(self): @@ -129,6 +132,8 @@ def from_row(cls, row): @dataclass class Tool: + "A tool that can be called by a model." + name: str description: Optional[str] = None input_schema: Dict = field(default_factory=dict) @@ -293,6 +298,8 @@ async def prepare_async(self): @dataclass class ToolCall: + "A request by the model to call a tool." + name: str arguments: dict tool_call_id: Optional[str] = None @@ -300,6 +307,8 @@ class ToolCall: @dataclass class ToolResult: + "The result of executing a tool call." + name: str output: str attachments: List[Attachment] = field(default_factory=list) @@ -374,10 +383,12 @@ def __init__( @property def prompt(self): + "The text of the prompt, with any fragments concatenated." return "\n".join(self.fragments + ([self._prompt] if self._prompt else [])) @property def system(self): + "The system prompt, with any system fragments concatenated." bits = [ bit.strip() for bit in (self.system_fragments + [self._system or ""]) @@ -2086,6 +2097,8 @@ def __repr__(self) -> str: @dataclass class ModelWithAliases: + "A model with its optional async counterpart and aliases." + model: Model async_model: AsyncModel aliases: Set[str] diff --git a/llm/templates.py b/llm/templates.py index 657a47641..ac1b7c716 100644 --- a/llm/templates.py +++ b/llm/templates.py @@ -9,6 +9,8 @@ class AttachmentType(BaseModel): class Template(BaseModel): + """A reusable prompt template.""" + name: str prompt: Optional[str] = None system: Optional[str] = None @@ -39,6 +41,7 @@ def __init__(self, **data): def evaluate( self, input: str, params: Optional[Dict[str, Any]] = None ) -> Tuple[Optional[str], Optional[str]]: + """Evaluate the template with the given input and parameters, returning (prompt, system).""" params = params or {} params["input"] = input if self.defaults: @@ -56,6 +59,7 @@ def evaluate( return prompt, system def vars(self) -> set: + """Return the set of variable names used in the prompt and system templates.""" all_vars = set() for text in [self.prompt, self.system]: if not text: @@ -65,6 +69,7 @@ def vars(self) -> set: @classmethod def interpolate(cls, text: Optional[str], params: Dict[str, Any]) -> Optional[str]: + """Substitute template variables in text with values from params, raising MissingVariables if any are absent.""" if not text: return text # Confirm all variables in text are provided @@ -79,6 +84,7 @@ def interpolate(cls, text: Optional[str], params: Dict[str, Any]) -> Optional[st @staticmethod def extract_vars(string_template: string.Template) -> List[str]: + """Extract and return the list of named variable identifiers from a string.Template.""" return [ match.group("named") for match in string_template.pattern.finditer(string_template.template) From 62b8864fda8995958133c2924d851e2ea1f30c58 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 31 Mar 2026 13:32:04 -0700 Subject: [PATCH 011/258] Release 0.30 --- docs/changelog.md | 8 +++++++- pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 321e7af79..4ae538cbf 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,7 +1,13 @@ # Changelog +(v0_30)= +## 0.30 (2026-03-31) + +- The {ref}`register_models() plugin hook ` now takes an optional `model_aliases` parameter listing all of the models, async models and aliases that have been registered so far by other plugins. A plugin with `@hookimpl(trylast=True)` can use this to take previously registered models into account. +- Added docstrings to public classes and methods and included those directly in the documentation. + (v0_29)= -## 0.29 (2025-03-17) +## 0.29 (2026-03-17) - The `-t/--template` option now works correctly with the `-x/--extract` and `--xl/--extract-last` flags. - `llm logs` now shows any additional model options in the Markdown output. [#1322](https://github.com/simonw/llm/issues/1322) diff --git a/pyproject.toml b/pyproject.toml index 77e92f89b..a9ae3b6dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm" -version = "0.29" +version = "0.30" description = "CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine." readme = { file = "README.md", content-type = "text/markdown" } authors = [ From 7169fe9085e63a98dee4374bda5848f5f9b1363c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 31 Mar 2026 13:33:56 -0700 Subject: [PATCH 012/258] Add Usage to llm/__init__ __all__ (to fix Ruff) --- docs/fragments.md | 2 +- llm/__init__.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/fragments.md b/docs/fragments.md index c623545b7..f281fb063 100644 --- a/docs/fragments.md +++ b/docs/fragments.md @@ -30,7 +30,7 @@ from importlib.metadata import version llm_version = version("llm") cog.out(f'The URL will be fetched with the user-agent `llm/{llm_version} (https://llm.datasette.io/)`.') ]]]--> -The URL will be fetched with the user-agent `llm/0.29 (https://llm.datasette.io/)`. +The URL will be fetched with the user-agent `llm/0.30 (https://llm.datasette.io/)`. The `-f` option can be used multiple times to combine together multiple fragments. diff --git a/llm/__init__.py b/llm/__init__.py index 27248b336..0e54cc8c0 100644 --- a/llm/__init__.py +++ b/llm/__init__.py @@ -65,6 +65,7 @@ "ToolCall", "ToolOutput", "ToolResult", + "Usage", "user_dir", "schema_dsl", ] From f2c7a2a8074fa4a343b95eb737137d4791cef033 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 31 Mar 2026 13:45:24 -0700 Subject: [PATCH 013/258] Link to #1389 from changelog --- docs/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 4ae538cbf..3f4241773 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,7 +3,7 @@ (v0_30)= ## 0.30 (2026-03-31) -- The {ref}`register_models() plugin hook ` now takes an optional `model_aliases` parameter listing all of the models, async models and aliases that have been registered so far by other plugins. A plugin with `@hookimpl(trylast=True)` can use this to take previously registered models into account. +- The {ref}`register_models() plugin hook ` now takes an optional `model_aliases` parameter listing all of the models, async models and aliases that have been registered so far by other plugins. A plugin with `@hookimpl(trylast=True)` can use this to take previously registered models into account. [#1389](https://github.com/simonw/llm/issues/1389) - Added docstrings to public classes and methods and included those directly in the documentation. (v0_29)= From cad03fb4f4a3bca969ed2583102593c87688f5b6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 4 Apr 2026 07:06:46 -0700 Subject: [PATCH 014/258] Register async models for extra-openai-models.yaml, closes #1395 Note that Completion models do not have an async class so will not be registered as async. --- llm/default_plugins/openai_models.py | 13 +++++++++++-- tests/test_llm.py | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 849d5fde2..55fa7a38d 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -294,10 +294,12 @@ def register_models(register): kwargs["audio"] = True if extra_model.get("completion"): klass = Completion + async_klass = None else: klass = Chat - chat_model = klass( - model_id, + async_klass = AsyncChat + model_kwargs = dict( + model_id=model_id, model_name=model_name, api_base=api_base, api_type=api_type, @@ -307,12 +309,19 @@ def register_models(register): reasoning=reasoning, **kwargs, ) + chat_model = klass(**model_kwargs) + async_model = async_klass(**model_kwargs) if async_klass else None if api_base: chat_model.needs_key = None + if async_model: + async_model.needs_key = None if extra_model.get("api_key_name"): chat_model.needs_key = extra_model["api_key_name"] + if async_model: + async_model.needs_key = extra_model["api_key_name"] register( chat_model, + async_model, aliases=aliases, ) diff --git a/tests/test_llm.py b/tests/test_llm.py index 8b307cc2a..526cc1258 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -431,6 +431,22 @@ def test_openai_localai_configuration(mocked_localai, user_path): } +def test_extra_openai_models_async(user_path): + from llm.default_plugins.openai_models import AsyncChat + + config_path = user_path / "extra-openai-models.yaml" + config_path.write_text(EXTRA_MODELS_YAML, "utf-8") + async_model = llm.get_async_model("orca") + assert isinstance(async_model, AsyncChat) + assert async_model.model_id == "orca" + assert async_model.model_name == "orca-mini-3b" + assert async_model.api_base == "http://localai.localhost" + assert async_model.needs_key is None + # Completion models should not have an async variant + with pytest.raises(llm.UnknownModelError): + llm.get_async_model("completion-babbage") + + @pytest.mark.parametrize( "args,exit_code", ( From 1bfb96d60b7724b75cd67eb4b0bc470153e9dac2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Apr 2026 09:11:44 -0700 Subject: [PATCH 015/258] Ignore *.db (temporary test databases) --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index aa1fee1f0..583b6c6e2 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ venv .DS_Store .idea/ .vscode/ -uv.lock \ No newline at end of file +uv.lock +*.db From 3da80543fe634188ce6d154b5cab1645a2162f9c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 20 Apr 2026 09:27:32 -0700 Subject: [PATCH 016/258] Phase 1: Part + Message value types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add llm/parts.py with Part, TextPart, ReasoningPart, ToolCallPart, ToolResultPart, AttachmentPart, Message, and StreamEvent dataclasses. Parts round-trip through to_dict/from_dict (attachments base64-encoded); role lives on Message, not on Part. Add user/assistant/system/ tool_message constructor helpers that accept strings, Attachments, Parts, and nested lists. Everything is a pure value — identity belongs to storage, which lives elsewhere. No Response integration yet; that's Phase 2. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/__init__.py | 28 +++- llm/parts.py | 330 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_parts.py | 311 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 668 insertions(+), 1 deletion(-) create mode 100644 llm/parts.py create mode 100644 tests/test_parts.py diff --git a/llm/__init__.py b/llm/__init__.py index 0e54cc8c0..bb84c3911 100644 --- a/llm/__init__.py +++ b/llm/__init__.py @@ -26,6 +26,20 @@ ToolResult, Usage, ) +from .parts import ( + AttachmentPart, + Message, + Part, + ReasoningPart, + StreamEvent, + TextPart, + ToolCallPart, + ToolResultPart, + assistant, + system, + tool_message, + user, +) from .utils import schema_dsl, Fragment from .embeddings import Collection from .templates import Template @@ -43,7 +57,9 @@ "AsyncKeyModel", "AsyncModel", "AsyncResponse", + "assistant", "Attachment", + "AttachmentPart", "CancelToolCall", "Collection", "Conversation", @@ -53,21 +69,31 @@ "get_model", "hookimpl", "KeyModel", + "Message", "Model", "ModelError", "NeedsKeyException", "Options", + "Part", "Prompt", + "ReasoningPart", "Response", + "schema_dsl", + "StreamEvent", + "system", "Template", + "TextPart", "Tool", "Toolbox", "ToolCall", + "ToolCallPart", + "tool_message", "ToolOutput", "ToolResult", + "ToolResultPart", "Usage", + "user", "user_dir", - "schema_dsl", ] DEFAULT_MODEL = "gpt-4o-mini" diff --git a/llm/parts.py b/llm/parts.py new file mode 100644 index 000000000..7a1a83a80 --- /dev/null +++ b/llm/parts.py @@ -0,0 +1,330 @@ +"""Part, Message, and StreamEvent value types. + +Parts represent the structured content of model interactions: text, +reasoning, tool calls, tool results, and attachments. A Message wraps a +list of Parts with a role. StreamEvent wraps a streaming chunk with type +information so consumers can distinguish text from reasoning from tool +call fragments as they arrive. + +These types are pure values — identity (ids, parent links, storage keys) +is a storage concern that lives elsewhere. Two Messages with identical +content are equal. +""" + +import base64 +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from .models import Attachment + + +def _attachment_to_dict(att: Attachment) -> Dict[str, Any]: + d: Dict[str, Any] = {} + if att.type: + d["type"] = att.type + if att.url: + d["url"] = att.url + if att.path: + d["path"] = att.path + if att.content: + d["content"] = base64.b64encode(att.content).decode("ascii") + return d + + +def _attachment_from_dict(d: Dict[str, Any]) -> Attachment: + content = d.get("content") + if isinstance(content, str): + content = base64.b64decode(content) + return Attachment( + type=d.get("type"), + path=d.get("path"), + url=d.get("url"), + content=content, + ) + + +@dataclass +class Part: + """Base class for all parts. Role lives on the enclosing Message.""" + + def to_dict(self) -> Dict[str, Any]: + raise NotImplementedError + + @staticmethod + def from_dict(d: Dict[str, Any]) -> "Part": + type_ = d.get("type") + pm = d.get("provider_metadata") + if type_ == "text": + return TextPart(text=d.get("text", ""), provider_metadata=pm) + if type_ == "reasoning": + return ReasoningPart( + text=d.get("text", ""), + redacted=d.get("redacted", False), + token_count=d.get("token_count"), + provider_metadata=pm, + ) + if type_ == "tool_call": + return ToolCallPart( + name=d["name"], + arguments=d.get("arguments", {}), + tool_call_id=d.get("tool_call_id"), + server_executed=d.get("server_executed", False), + provider_metadata=pm, + ) + if type_ == "tool_result": + return ToolResultPart( + name=d["name"], + output=d.get("output", ""), + tool_call_id=d.get("tool_call_id"), + server_executed=d.get("server_executed", False), + exception=d.get("exception"), + attachments=[ + _attachment_from_dict(a) for a in d.get("attachments", []) + ], + provider_metadata=pm, + ) + if type_ == "attachment": + att_dict = d.get("attachment") + attachment = _attachment_from_dict(att_dict) if att_dict else None + return AttachmentPart(attachment=attachment, provider_metadata=pm) + raise ValueError(f"Unknown part type: {type_!r}") + + +@dataclass +class TextPart(Part): + text: str = "" + provider_metadata: Optional[Dict[str, Any]] = None + + def to_dict(self) -> Dict[str, Any]: + d: Dict[str, Any] = {"type": "text", "text": self.text} + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d + + +@dataclass +class ReasoningPart(Part): + """Reasoning/thinking tokens from the model. + + `redacted=True, text=""` represents the opaque-token-count case + (OpenAI GPT-5 series, Gemini) where the provider reports only a + count, not content. + """ + + text: str = "" + redacted: bool = False + token_count: Optional[int] = None + provider_metadata: Optional[Dict[str, Any]] = None + + def to_dict(self) -> Dict[str, Any]: + d: Dict[str, Any] = {"type": "reasoning", "text": self.text} + if self.redacted: + d["redacted"] = True + if self.token_count is not None: + d["token_count"] = self.token_count + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d + + +@dataclass +class ToolCallPart(Part): + """A request by the model to call a tool. + + `server_executed=True` marks calls the provider executed on the + server (Anthropic web search, Gemini code execution) rather than + the LLM tool framework. + """ + + name: str = "" + arguments: Dict[str, Any] = field(default_factory=dict) + tool_call_id: Optional[str] = None + server_executed: bool = False + provider_metadata: Optional[Dict[str, Any]] = None + + def to_dict(self) -> Dict[str, Any]: + d: Dict[str, Any] = { + "type": "tool_call", + "name": self.name, + "arguments": self.arguments, + } + if self.tool_call_id is not None: + d["tool_call_id"] = self.tool_call_id + if self.server_executed: + d["server_executed"] = True + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d + + +@dataclass +class ToolResultPart(Part): + """The result of a tool call.""" + + name: str = "" + output: str = "" + tool_call_id: Optional[str] = None + server_executed: bool = False + attachments: List[Any] = field(default_factory=list) + exception: Optional[str] = None + provider_metadata: Optional[Dict[str, Any]] = None + + def to_dict(self) -> Dict[str, Any]: + d: Dict[str, Any] = { + "type": "tool_result", + "name": self.name, + "output": self.output, + } + if self.tool_call_id is not None: + d["tool_call_id"] = self.tool_call_id + if self.server_executed: + d["server_executed"] = True + if self.exception is not None: + d["exception"] = self.exception + if self.attachments: + d["attachments"] = [_attachment_to_dict(a) for a in self.attachments] + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d + + +@dataclass +class AttachmentPart(Part): + """An inline attachment (image, audio, file).""" + + attachment: Optional[Attachment] = None + provider_metadata: Optional[Dict[str, Any]] = None + + def to_dict(self) -> Dict[str, Any]: + d: Dict[str, Any] = {"type": "attachment"} + if self.attachment: + d["attachment"] = _attachment_to_dict(self.attachment) + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d + + +@dataclass +class Message: + """A single turn in a conversation: role + list of parts. + + `parts` contains one or more Part objects. `provider_metadata` + carries opaque provider-specific data attached to the message as a + whole; part-level data lives on the individual Part's + `provider_metadata`. + """ + + role: str + parts: List[Part] = field(default_factory=list) + provider_metadata: Optional[Dict[str, Any]] = None + + def to_dict(self) -> Dict[str, Any]: + d: Dict[str, Any] = { + "role": self.role, + "parts": [p.to_dict() for p in self.parts], + } + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d + + @staticmethod + def from_dict(d: Dict[str, Any]) -> "Message": + return Message( + role=d["role"], + parts=[Part.from_dict(p) for p in d.get("parts", [])], + provider_metadata=d.get("provider_metadata"), + ) + + +def normalize_parts(items: Any) -> List[Part]: + """Normalize helper inputs to a list of Part objects. + + Accepts str (→ TextPart), Attachment (→ AttachmentPart), Part + (passed through), or a list/tuple of those (flattened one level). + """ + out: List[Part] = [] + for item in items: + if isinstance(item, Part): + out.append(item) + elif isinstance(item, str): + out.append(TextPart(text=item)) + elif isinstance(item, Attachment): + out.append(AttachmentPart(attachment=item)) + elif isinstance(item, (list, tuple)): + out.extend(normalize_parts(item)) + else: + raise TypeError(f"Cannot convert {item!r} to an llm Part") + return out + + +def system( + *items: Any, provider_metadata: Optional[Dict[str, Any]] = None +) -> Message: + "Build a Message with role='system'." + return Message( + role="system", + parts=normalize_parts(items), + provider_metadata=provider_metadata, + ) + + +def user( + *items: Any, provider_metadata: Optional[Dict[str, Any]] = None +) -> Message: + "Build a Message with role='user'." + return Message( + role="user", + parts=normalize_parts(items), + provider_metadata=provider_metadata, + ) + + +def assistant( + *items: Any, provider_metadata: Optional[Dict[str, Any]] = None +) -> Message: + "Build a Message with role='assistant'." + return Message( + role="assistant", + parts=normalize_parts(items), + provider_metadata=provider_metadata, + ) + + +def tool_message( + *items: Any, provider_metadata: Optional[Dict[str, Any]] = None +) -> Message: + "Build a Message with role='tool' (typically wrapping ToolResultParts)." + return Message( + role="tool", + parts=normalize_parts(items), + provider_metadata=provider_metadata, + ) + + +@dataclass +class StreamEvent: + """A streaming event from a model response. + + `part_index` groups events into parts — events sharing an index + belong to the same logical part. Mixing families (e.g. text with + tool_call_name) at the same index is a plugin bug. + + `provider_metadata` carries opaque provider data (Anthropic + `signature`, Gemini `thoughtSignature`, OpenAI `encrypted_content`) + that must be echoed back on the next request; the framework merges + it onto the finalized Part (last non-None wins per top-level key). + + `message_index` is for providers that emit multiple assistant + messages in a single response (Anthropic server-side tool + execution); most plugins leave it at 0. + """ + + type: str # "text" / "reasoning" / "tool_call_name" / + # "tool_call_args" / "tool_result" + chunk: str + part_index: int + tool_call_id: Optional[str] = None + server_executed: bool = False + tool_name: Optional[str] = None + provider_metadata: Optional[Dict[str, Any]] = None + message_index: int = 0 diff --git a/tests/test_parts.py b/tests/test_parts.py new file mode 100644 index 000000000..b5f4c8cc0 --- /dev/null +++ b/tests/test_parts.py @@ -0,0 +1,311 @@ +"""Tests for Part, Message, StreamEvent and the constructor helpers. + +Phase 1 covers the in-memory value types and JSON round-trip only. +No Response / streaming / plugin integration yet. +""" + +import json +import pytest + +import llm + + +# -- Exports ------------------------------------------------------------ + + +class TestExports: + def test_llm_exports_part_types(self): + assert llm.Part is not None + assert llm.TextPart is not None + assert llm.ReasoningPart is not None + assert llm.ToolCallPart is not None + assert llm.ToolResultPart is not None + assert llm.AttachmentPart is not None + assert llm.Message is not None + assert llm.StreamEvent is not None + + def test_llm_exports_constructor_helpers(self): + assert callable(llm.user) + assert callable(llm.assistant) + assert callable(llm.system) + assert callable(llm.tool_message) + + +# -- Part subclasses ---------------------------------------------------- + + +class TestTextPart: + def test_roundtrip(self): + part = llm.TextPart(text="Hello world") + restored = llm.Part.from_dict(part.to_dict()) + assert restored == part + assert isinstance(restored, llm.TextPart) + assert restored.text == "Hello world" + + def test_to_dict_shape(self): + assert llm.TextPart(text="hi").to_dict() == {"type": "text", "text": "hi"} + + def test_with_provider_metadata(self): + part = llm.TextPart( + text="hi", provider_metadata={"openai": {"flag": True}} + ) + restored = llm.Part.from_dict(part.to_dict()) + assert restored == part + + +class TestReasoningPart: + def test_roundtrip_with_text(self): + part = llm.ReasoningPart(text="Let me think...") + restored = llm.Part.from_dict(part.to_dict()) + assert restored == part + assert restored.text == "Let me think..." + assert restored.redacted is False + assert restored.token_count is None + + def test_roundtrip_redacted(self): + part = llm.ReasoningPart(text="", redacted=True, token_count=150) + d = part.to_dict() + assert d["redacted"] is True + assert d["token_count"] == 150 + restored = llm.Part.from_dict(d) + assert restored == part + + +class TestToolCallPart: + def test_roundtrip(self): + part = llm.ToolCallPart( + name="search", + arguments={"query": "weather"}, + tool_call_id="call_123", + ) + restored = llm.Part.from_dict(part.to_dict()) + assert restored == part + assert restored.server_executed is False + + def test_server_executed_flag_roundtrips(self): + part = llm.ToolCallPart( + name="web_search", + arguments={"q": "x"}, + tool_call_id="c1", + server_executed=True, + ) + d = part.to_dict() + assert d["server_executed"] is True + restored = llm.Part.from_dict(d) + assert restored.server_executed is True + + +class TestToolResultPart: + def test_roundtrip(self): + part = llm.ToolResultPart( + name="search", output="72F sunny", tool_call_id="c1" + ) + restored = llm.Part.from_dict(part.to_dict()) + assert restored == part + assert restored.exception is None + assert restored.attachments == [] + + def test_with_exception(self): + part = llm.ToolResultPart( + name="t", output="", tool_call_id="c1", exception="boom" + ) + restored = llm.Part.from_dict(part.to_dict()) + assert restored.exception == "boom" + + +class TestAttachmentPart: + def test_roundtrip_with_url(self): + att = llm.Attachment(url="http://example.com/cat.jpg") + part = llm.AttachmentPart(attachment=att) + restored = llm.Part.from_dict(part.to_dict()) + assert isinstance(restored, llm.AttachmentPart) + assert restored.attachment.url == "http://example.com/cat.jpg" + + def test_roundtrip_with_path(self): + att = llm.Attachment(type="image/jpeg", path="/tmp/x.jpg") + part = llm.AttachmentPart(attachment=att) + restored = llm.Part.from_dict(part.to_dict()) + assert restored.attachment.path == "/tmp/x.jpg" + assert restored.attachment.type == "image/jpeg" + + def test_roundtrip_with_bytes_uses_base64(self): + raw = b"\x89PNG fake bytes" + att = llm.Attachment(type="image/png", content=raw) + part = llm.AttachmentPart(attachment=att) + d = part.to_dict() + # Content must be a base64-encoded string in the dict form + assert isinstance(d["attachment"]["content"], str) + import base64 + + assert base64.b64decode(d["attachment"]["content"]) == raw + # And round-trip back to the original bytes + restored = llm.Part.from_dict(d) + assert restored.attachment.content == raw + + def test_json_serializable(self): + att = llm.Attachment(type="image/png", content=b"\x00\x01\x02") + part = llm.AttachmentPart(attachment=att) + # Must survive json dumps/loads + restored = llm.Part.from_dict(json.loads(json.dumps(part.to_dict()))) + assert restored.attachment.content == b"\x00\x01\x02" + + +class TestUnknownPart: + def test_from_dict_unknown_type_raises(self): + with pytest.raises(ValueError): + llm.Part.from_dict({"type": "nonsense"}) + + +class TestRoleNotOnPart: + def test_text_part_has_no_role_attribute(self): + # Role lives on Message. Parts are content-only. + part = llm.TextPart(text="hi") + assert not hasattr(part, "role") + + def test_reasoning_part_has_no_role_attribute(self): + assert not hasattr(llm.ReasoningPart(text=""), "role") + + def test_tool_call_part_has_no_role_attribute(self): + assert not hasattr( + llm.ToolCallPart(name="t", arguments={}, tool_call_id="c1"), + "role", + ) + + +# -- Message ------------------------------------------------------------ + + +class TestMessage: + def test_roundtrip_simple_user_message(self): + m = llm.Message(role="user", parts=[llm.TextPart(text="hi")]) + restored = llm.Message.from_dict(m.to_dict()) + assert restored == m + + def test_roundtrip_with_provider_metadata(self): + m = llm.Message( + role="assistant", + parts=[llm.TextPart(text="hi")], + provider_metadata={"anthropic": {"signature": "abc"}}, + ) + restored = llm.Message.from_dict(m.to_dict()) + assert restored == m + + def test_roundtrip_mixed_parts(self): + m = llm.Message( + role="assistant", + parts=[ + llm.ReasoningPart(text="Thinking"), + llm.TextPart(text="Result"), + llm.ToolCallPart( + name="search", + arguments={"q": "x"}, + tool_call_id="c1", + ), + ], + ) + restored = llm.Message.from_dict(m.to_dict()) + assert restored == m + + def test_empty_provider_metadata_omitted(self): + m = llm.Message(role="user", parts=[llm.TextPart(text="x")]) + d = m.to_dict() + assert "provider_metadata" not in d + + def test_none_and_empty_provider_metadata_equivalent(self): + m_none = llm.Message(role="user", parts=[llm.TextPart(text="x")]) + m_empty = llm.Message( + role="user", + parts=[llm.TextPart(text="x")], + provider_metadata={}, + ) + # Both serialize the same (empty metadata is omitted) + assert m_none.to_dict() == m_empty.to_dict() + + +# -- Constructor helpers ----------------------------------------------- + + +class TestHelpers: + def test_user_with_string(self): + m = llm.user("hi") + assert m.role == "user" + assert m.parts == [llm.TextPart(text="hi")] + + def test_assistant_with_string(self): + m = llm.assistant("there") + assert m.role == "assistant" + assert m.parts == [llm.TextPart(text="there")] + + def test_system_with_string(self): + m = llm.system("be brief") + assert m.role == "system" + assert m.parts == [llm.TextPart(text="be brief")] + + def test_tool_message_with_part(self): + tr = llm.ToolResultPart(name="t", output="r", tool_call_id="c1") + m = llm.tool_message(tr) + assert m.role == "tool" + assert m.parts == [tr] + + def test_helper_accepts_attachment(self): + att = llm.Attachment(url="http://example.com/x.jpg") + m = llm.user("describe this", att) + assert m.parts == [ + llm.TextPart(text="describe this"), + llm.AttachmentPart(attachment=att), + ] + + def test_helper_accepts_existing_part(self): + tp = llm.TextPart(text="pre-built") + m = llm.user(tp) + assert m.parts == [tp] + + def test_helper_flattens_one_level(self): + # Nested list gets flattened one level. + m = llm.user(["one", "two"], "three") + assert m.parts == [ + llm.TextPart(text="one"), + llm.TextPart(text="two"), + llm.TextPart(text="three"), + ] + + def test_helper_rejects_unknown_types(self): + with pytest.raises(TypeError): + llm.user(42) + + def test_helper_with_provider_metadata(self): + m = llm.assistant("hi", provider_metadata={"openai": {"id": "x"}}) + assert m.provider_metadata == {"openai": {"id": "x"}} + + +# -- StreamEvent (type only, no Response integration yet) -------------- + + +class TestStreamEvent: + def test_dataclass_defaults(self): + ev = llm.StreamEvent(type="text", chunk="hi", part_index=0) + assert ev.type == "text" + assert ev.chunk == "hi" + assert ev.part_index == 0 + assert ev.tool_call_id is None + assert ev.server_executed is False + assert ev.tool_name is None + assert ev.provider_metadata is None + assert ev.message_index == 0 + + def test_all_fields_accepted(self): + ev = llm.StreamEvent( + type="tool_call_args", + chunk='{"q":', + part_index=2, + tool_call_id="c1", + server_executed=True, + tool_name="search", + provider_metadata={"openai": {"x": 1}}, + message_index=1, + ) + assert ev.tool_call_id == "c1" + assert ev.server_executed is True + assert ev.tool_name == "search" + assert ev.provider_metadata == {"openai": {"x": 1}} + assert ev.message_index == 1 From fc7fb28f889ed3a662db71ee4cecf69fc80bda32 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 20 Apr 2026 09:31:30 -0700 Subject: [PATCH 017/258] Phase 2: Response streaming scaffolding (stream_events, messages) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Teach Response and AsyncResponse to accept str | StreamEvent from execute(). Plain-str plugins keep working unchanged — their yields are wrapped as StreamEvent(type="text", chunk=..., part_index=0) internally. New capabilities: - response.stream_events() / response.astream_events() yield every event (text, reasoning, tool_call_*, tool_result) as the model produces it. Iteration ("for chunk in response") still yields only text strings. - response.messages returns the list of assembled Message objects once the response is done. AsyncResponse.messages raises if not yet awaited. - _BaseResponse._build_parts() groups events by part_index into typed Parts (TextPart, ReasoningPart, ToolCallPart, ToolResultPart). Mixing families at the same index raises ValueError. - Opaque reasoning token counts: plugins set response._reasoning_token_count = N; _build_parts prepends a ReasoningPart(redacted=True, token_count=N, text=""). - provider_metadata merges across events for the same part (last non-None wins per top-level namespace key). ChainResponse.stream_events and AsyncChainResponse.astream_events pass through from each underlying response. 530 tests passing; no regressions to the existing suite. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/models.py | 351 +++++++++++++++++++++++++++++++++++++++----- tests/test_parts.py | 333 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 648 insertions(+), 36 deletions(-) diff --git a/llm/models.py b/llm/models.py index 892a4f2ba..003fe0176 100644 --- a/llm/models.py +++ b/llm/models.py @@ -674,6 +674,16 @@ def __init__( self.stream = stream self._key = key self._chunks: List[str] = [] + # Every StreamEvent ever yielded by execute(), in order. Plain + # str yields are wrapped as StreamEvent(type="text", part_index=0) + # so this buffer is the single source of truth for replay and + # for assembling response.messages. + self._stream_events: List[Any] = [] + # Plugins set this when the provider reports an opaque reasoning + # token count (no streamed reasoning text). _build_parts() + # prepends a ReasoningPart(redacted=True, token_count=N) when + # non-zero. + self._reasoning_token_count: int = 0 self._done = False self._tool_calls: List[ToolCall] = [] self.response_json: Optional[Dict[str, Any]] = None @@ -693,6 +703,164 @@ def __init__( if self.prompt.tools and not self.model.supports_tools: raise ValueError(f"{self.model} does not support tools") + def _process_chunk(self, chunk): + """Normalize a chunk from execute() into a StreamEvent and return + the text str (or None) that __iter__ should yield. + + Plain str yields from legacy plugins are wrapped as text events + at part_index=0. Side effects: populates self._stream_events and + self._chunks. + """ + from .parts import StreamEvent + + if isinstance(chunk, StreamEvent): + self._stream_events.append(chunk) + if chunk.type == "text": + self._chunks.append(chunk.chunk) + return chunk.chunk + return None + # Legacy plain-str plugin. + event = StreamEvent(type="text", chunk=chunk, part_index=0) + self._stream_events.append(event) + self._chunks.append(chunk) + return chunk + + def _build_parts(self) -> List[Any]: + """Assemble Part objects from the accumulated stream events. + + Events sharing a part_index group into one Part. Mixing + families (text vs tool_call vs reasoning vs tool_result) at the + same index is a plugin bug — raises ValueError instead of + silently dropping content. + """ + from .parts import ( + ReasoningPart, + TextPart, + ToolCallPart, + ToolResultPart, + ) + + def family(t: str) -> str: + if t in ("tool_call_name", "tool_call_args"): + return "tool_call" + return t + + parts: List[Any] = [] + current_index: Optional[int] = None + current_family: Optional[str] = None + text_buf: List[str] = [] + tool_name: Optional[str] = None + tool_args_buf: List[str] = [] + tool_call_id: Optional[str] = None + server_executed = False + tool_result_name: Optional[str] = None + pm_merged: Optional[Dict[str, Any]] = None + + def finalize(): + nonlocal pm_merged + if current_family is None: + return + if current_family == "text": + text = "".join(text_buf) + if text: + parts.append(TextPart(text=text, provider_metadata=pm_merged)) + elif current_family == "reasoning": + text = "".join(text_buf) + if text: + parts.append( + ReasoningPart(text=text, provider_metadata=pm_merged) + ) + elif current_family == "tool_call": + args_str = "".join(tool_args_buf) + try: + arguments = json.loads(args_str) if args_str else {} + except json.JSONDecodeError: + arguments = {"_raw": args_str} + parts.append( + ToolCallPart( + name=tool_name or "", + arguments=arguments, + tool_call_id=tool_call_id, + server_executed=server_executed, + provider_metadata=pm_merged, + ) + ) + elif current_family == "tool_result": + parts.append( + ToolResultPart( + name=tool_result_name or "", + output="".join(text_buf), + tool_call_id=tool_call_id, + server_executed=server_executed, + provider_metadata=pm_merged, + ) + ) + + for event in self._stream_events: + ev_family = family(event.type) + if event.part_index != current_index: + finalize() + current_index = event.part_index + current_family = ev_family + text_buf = [] + tool_name = None + tool_args_buf = [] + tool_call_id = None + server_executed = False + tool_result_name = None + pm_merged = None + elif current_family is not None and ev_family != current_family: + raise ValueError( + f"StreamEvent type {event.type!r} is incompatible with " + f"prior type at part_index={event.part_index}. " + "Allocate a new part_index for a different content type." + ) + + if event.type == "text": + text_buf.append(event.chunk) + elif event.type == "reasoning": + text_buf.append(event.chunk) + elif event.type == "tool_call_name": + tool_name = (tool_name or "") + event.chunk + if event.tool_call_id: + tool_call_id = event.tool_call_id + if event.server_executed: + server_executed = True + elif event.type == "tool_call_args": + tool_args_buf.append(event.chunk) + if event.tool_call_id and tool_call_id is None: + tool_call_id = event.tool_call_id + if event.server_executed: + server_executed = True + elif event.type == "tool_result": + text_buf.append(event.chunk) + if event.tool_call_id and tool_call_id is None: + tool_call_id = event.tool_call_id + if event.server_executed: + server_executed = True + if event.tool_name: + tool_result_name = event.tool_name + + if event.provider_metadata: + merged = dict(pm_merged) if pm_merged else {} + for k, v in event.provider_metadata.items(): + merged[k] = v + pm_merged = merged + + finalize() + + if self._reasoning_token_count: + parts.insert( + 0, + ReasoningPart( + text="", + redacted=True, + token_count=self._reasoning_token_count, + ), + ) + + return parts + def add_tool_call(self, tool_call: ToolCall): self._tool_calls.append(tool_call) @@ -1177,43 +1345,93 @@ def usage(self) -> Usage: details=self.token_details, ) - def __iter__(self) -> Iterator[str]: - self._start = time.monotonic() - self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) - if self._done: - yield from self._chunks - return - + def _iter_events(self): + """Drive self.model.execute() once. Yields every chunk it + produces, each already appended to self._stream_events by + _process_chunk as a side effect. + """ if isinstance(self.model, Model): - for chunk in self.model.execute( + generator = self.model.execute( self.prompt, stream=self.stream, response=self, conversation=self.conversation, - ): - assert chunk is not None - yield chunk - self._chunks.append(chunk) + ) elif isinstance(self.model, KeyModel): - for chunk in self.model.execute( + generator = self.model.execute( self.prompt, stream=self.stream, response=self, conversation=self.conversation, key=self.model.get_key(self._key), - ): - assert chunk is not None - yield chunk - self._chunks.append(chunk) + ) else: raise Exception("self.model must be a Model or KeyModel") + for chunk in generator: + assert chunk is not None + yield chunk + + def __iter__(self) -> Iterator[str]: + self._start = time.monotonic() + self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) + if self._done: + yield from self._chunks + return + + for chunk in self._iter_events(): + text = self._process_chunk(chunk) + if text is not None: + yield text + if self.conversation: self.conversation.responses.append(self) self._end = time.monotonic() self._done = True self._on_done() + def stream_events(self): + """Yield StreamEvent objects as the model produces them. + + Whichever of __iter__ and stream_events runs first during live + streaming consumes the underlying generator. After completion, + both work — each replays from its own buffer. + """ + if self._done: + yield from self._stream_events + return + + self._start = time.monotonic() + self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) + for chunk in self._iter_events(): + # _process_chunk appends to self._stream_events; use it as + # the canonical source for what to yield so the replay path + # matches the live path byte-for-byte. + self._process_chunk(chunk) + yield self._stream_events[-1] + + if self.conversation: + self.conversation.responses.append(self) + self._end = time.monotonic() + self._done = True + self._on_done() + + @property + def messages(self) -> List[Any]: + """List of Message objects produced by this response. + + Almost always a single assistant Message; multiple messages are + possible for providers that emit multi-message responses during + server-side tool execution (not in this phase's scope). + """ + from .parts import Message + + self._force() + parts = self._build_parts() + if not parts: + return [] + return [Message(role="assistant", parts=parts)] + def __repr__(self): text = "... not yet done ..." if self._done: @@ -1416,12 +1634,7 @@ def __aiter__(self): self._iter_chunks = list(self._chunks) # Make a copy for iteration return self - async def __anext__(self) -> str: - if self._done: - if hasattr(self, "_iter_chunks") and self._iter_chunks: - return self._iter_chunks.pop(0) - raise StopAsyncIteration - + def _ensure_async_generator(self): if not hasattr(self, "_generator"): if isinstance(self.model, AsyncModel): self._generator = self.model.execute( @@ -1441,20 +1654,75 @@ async def __anext__(self) -> str: else: raise ValueError("self.model must be an AsyncModel or AsyncKeyModel") - try: - chunk = await self._generator.__anext__() + async def _async_finalize(self): + if self.conversation: + self.conversation.responses.append(self) + self._end = time.monotonic() + self._done = True + if hasattr(self, "_generator"): + del self._generator + await self._on_done() + + async def __anext__(self) -> str: + if self._done: + if hasattr(self, "_iter_chunks") and self._iter_chunks: + return self._iter_chunks.pop(0) + raise StopAsyncIteration + + self._ensure_async_generator() + # Skip non-text events — iteration yields only text. Loop until + # we find a text chunk or the generator is exhausted. + while True: + try: + chunk = await self._generator.__anext__() + except StopAsyncIteration: + await self._async_finalize() + raise assert chunk is not None - self._chunks.append(chunk) - return chunk - except StopAsyncIteration: - if self.conversation: - self.conversation.responses.append(self) - self._end = time.monotonic() - self._done = True - if hasattr(self, "_generator"): - del self._generator - await self._on_done() - raise + text = self._process_chunk(chunk) + if text is not None: + return text + + async def astream_events(self): + """Yield StreamEvent objects as the model produces them (async).""" + if self._done: + for event in self._stream_events: + yield event + return + + self._start = time.monotonic() + self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) + self._ensure_async_generator() + try: + while True: + try: + chunk = await self._generator.__anext__() + except StopAsyncIteration: + await self._async_finalize() + return + assert chunk is not None + self._process_chunk(chunk) + yield self._stream_events[-1] + finally: + pass + + @property + def messages(self) -> List[Any]: + """List of Message objects produced by this response. + + Raises ValueError if the response has not yet been awaited — + assembly depends on the full event stream. + """ + from .parts import Message + + if not self._done: + raise ValueError( + "Response not yet awaited — use 'await response' first" + ) + parts = self._build_parts() + if not parts: + return [] + return [Message(role="assistant", parts=parts)] async def _force(self): if not self._done: @@ -1670,6 +1938,11 @@ def __iter__(self) -> Iterator[str]: for response_item in self.responses(): yield from response_item + def stream_events(self): + "Yield StreamEvents from every response in the chain." + for response_item in self.responses(): + yield from response_item.stream_events() + def text(self) -> str: return "".join(self) @@ -1729,6 +2002,12 @@ async def __aiter__(self) -> AsyncIterator[str]: async for chunk in response_item: yield chunk + async def astream_events(self): + "Yield StreamEvents from every response in the chain." + async for response_item in self.responses(): + async for event in response_item.astream_events(): + yield event + async def text(self) -> str: all_chunks = [] async for chunk in self: diff --git a/tests/test_parts.py b/tests/test_parts.py index b5f4c8cc0..35ccc8f44 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -309,3 +309,336 @@ def test_all_fields_accepted(self): assert ev.tool_name == "search" assert ev.provider_metadata == {"openai": {"x": 1}} assert ev.message_index == 1 + + +# -- Phase 2: Response streaming scaffolding ---------------------------- +# +# Backward compat for plain-str plugins: iterating a Response still +# yields text strings, response.text() still works, self._chunks is +# still populated. +# +# New capabilities: +# - response.stream_events() / response.astream_events() +# - response.messages +# - _BaseResponse._build_parts() (internal, tested via .messages) + + +class TestPlainStrPluginCompat: + """A plugin that yields plain str must still work unchanged.""" + + def test_iter_yields_strings(self, mock_model): + mock_model.enqueue(["hello", " ", "world"]) + response = mock_model.prompt("hi") + chunks = list(response) + assert chunks == ["hello", " ", "world"] + + def test_text_returns_concatenation(self, mock_model): + mock_model.enqueue(["hello ", "world"]) + response = mock_model.prompt("hi") + assert response.text() == "hello world" + + def test_chunks_are_preserved(self, mock_model): + mock_model.enqueue(["a", "b", "c"]) + response = mock_model.prompt("hi") + response.text() + assert response._chunks == ["a", "b", "c"] + + +class TestStreamEventsFromPlainStrPlugin: + """When a plugin yields plain str, stream_events synthesizes text events.""" + + def test_stream_events_yields_text_events(self, mock_model): + mock_model.enqueue(["hel", "lo"]) + response = mock_model.prompt("hi") + events = list(response.stream_events()) + assert all(isinstance(e, llm.StreamEvent) for e in events) + assert [e.type for e in events] == ["text", "text"] + assert [e.chunk for e in events] == ["hel", "lo"] + assert all(e.part_index == 0 for e in events) + + def test_response_messages_is_single_assistant_text(self, mock_model): + mock_model.enqueue(["hello"]) + response = mock_model.prompt("hi") + response.text() + messages = response.messages + assert messages == [ + llm.Message(role="assistant", parts=[llm.TextPart(text="hello")]) + ] + + def test_empty_response_has_empty_messages(self, mock_model): + mock_model.enqueue([]) + response = mock_model.prompt("hi") + response.text() + assert response.messages == [] + + +class TestStreamEventsFromStreamEventPlugin: + """When a plugin yields StreamEvents, they pass through unchanged + and iteration filters to text only.""" + + def test_iter_yields_only_text_chunks(self, mock_model): + events = [ + llm.StreamEvent(type="reasoning", chunk="think ", part_index=0), + llm.StreamEvent(type="text", chunk="hel", part_index=1), + llm.StreamEvent(type="text", chunk="lo", part_index=1), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + chunks = list(response) + assert chunks == ["hel", "lo"] + + def test_stream_events_yields_all_events(self, mock_model): + events = [ + llm.StreamEvent(type="reasoning", chunk="t", part_index=0), + llm.StreamEvent(type="text", chunk="x", part_index=1), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + got = list(response.stream_events()) + assert [e.type for e in got] == ["reasoning", "text"] + + def test_messages_assembles_reasoning_then_text(self, mock_model): + events = [ + llm.StreamEvent(type="reasoning", chunk="thinking", part_index=0), + llm.StreamEvent(type="text", chunk="hello", part_index=1), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + assert response.messages == [ + llm.Message( + role="assistant", + parts=[ + llm.ReasoningPart(text="thinking"), + llm.TextPart(text="hello"), + ], + ) + ] + + def test_tool_call_name_and_args_merge(self, mock_model): + events = [ + llm.StreamEvent(type="text", chunk="calling", part_index=0), + llm.StreamEvent( + type="tool_call_name", + chunk="search", + part_index=1, + tool_call_id="c1", + ), + llm.StreamEvent( + type="tool_call_args", + chunk='{"q":', + part_index=1, + tool_call_id="c1", + ), + llm.StreamEvent( + type="tool_call_args", + chunk='"weather"}', + part_index=1, + tool_call_id="c1", + ), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + msgs = response.messages + assert len(msgs) == 1 + parts = msgs[0].parts + assert parts == [ + llm.TextPart(text="calling"), + llm.ToolCallPart( + name="search", + arguments={"q": "weather"}, + tool_call_id="c1", + ), + ] + + def test_tool_call_args_unparseable_json_falls_back(self, mock_model): + events = [ + llm.StreamEvent( + type="tool_call_name", + chunk="t", + part_index=0, + tool_call_id="c1", + ), + llm.StreamEvent( + type="tool_call_args", + chunk="not json", + part_index=0, + tool_call_id="c1", + ), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + part = response.messages[0].parts[0] + assert part.name == "t" + assert part.arguments == {"_raw": "not json"} + + def test_family_mismatch_at_same_part_index_raises(self, mock_model): + events = [ + llm.StreamEvent(type="text", chunk="x", part_index=0), + llm.StreamEvent( + type="tool_call_name", + chunk="t", + part_index=0, + tool_call_id="c1", + ), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + with pytest.raises(ValueError, match="part_index"): + response.messages # noqa: B018 + + def test_provider_metadata_merges_last_wins(self, mock_model): + events = [ + llm.StreamEvent( + type="reasoning", + chunk="think", + part_index=0, + provider_metadata={"anthropic": {"signature": "one"}}, + ), + llm.StreamEvent( + type="reasoning", + chunk="", + part_index=0, + provider_metadata={"anthropic": {"signature": "final"}}, + ), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + part = response.messages[0].parts[0] + assert part.provider_metadata == {"anthropic": {"signature": "final"}} + + def test_reasoning_token_count_prepends_redacted_part(self, mock_model): + # Plugin reports an opaque reasoning token count — framework + # prepends a ReasoningPart(redacted=True, token_count=N, text=""). + class CountingModel(type(mock_model)): + def execute(self, prompt, stream, response, conversation): + response._reasoning_token_count = 200 + yield llm.StreamEvent(type="text", chunk="hi", part_index=0) + + m = CountingModel() + response = m.prompt("x") + response.text() + parts = response.messages[0].parts + assert parts[0] == llm.ReasoningPart( + text="", redacted=True, token_count=200 + ) + assert parts[1] == llm.TextPart(text="hi") + + +class TestStreamEventsLiveDuringStreaming: + """Client code sees events arrive before the response is done — + this is the primary user-facing goal of this phase.""" + + def test_events_arrive_before_done(self, mock_model): + events = [ + llm.StreamEvent(type="reasoning", chunk="t", part_index=0), + llm.StreamEvent(type="text", chunk="hi", part_index=1), + ] + mock_model.enqueue(events) + response = mock_model.prompt("x") + seen = [] + for event in response.stream_events(): + # Record the _done state at the moment we receive the event. + seen.append((event.type, response._done)) + # Events arrived before _done was set. + assert [s[0] for s in seen] == ["reasoning", "text"] + assert all(not done for _type, done in seen) + # And after the generator is drained, the response is done. + assert response._done + + def test_stream_events_after_done_replays(self, mock_model): + mock_model.enqueue( + [llm.StreamEvent(type="text", chunk="hi", part_index=0)] + ) + response = mock_model.prompt("x") + first = list(response.stream_events()) + # Second call replays from the stored events. + second = list(response.stream_events()) + assert len(first) == 1 + assert [e.type for e in second] == ["text"] + assert [e.chunk for e in second] == ["hi"] + + def test_plain_str_stream_events_after_done_replays(self, mock_model): + mock_model.enqueue(["hello"]) + response = mock_model.prompt("x") + response.text() + events = list(response.stream_events()) + assert len(events) == 1 + assert events[0].type == "text" + assert events[0].chunk == "hello" + + +class TestAsyncStreamEvents: + @pytest.mark.asyncio + async def test_async_stream_events_live(self, async_mock_model): + events = [ + llm.StreamEvent(type="reasoning", chunk="r", part_index=0), + llm.StreamEvent(type="text", chunk="t", part_index=1), + ] + async_mock_model.enqueue(events) + response = async_mock_model.prompt("x") + seen_types = [] + async for event in response.astream_events(): + seen_types.append(event.type) + assert seen_types == ["reasoning", "text"] + + @pytest.mark.asyncio + async def test_async_iter_yields_only_text(self, async_mock_model): + events = [ + llm.StreamEvent(type="reasoning", chunk="r", part_index=0), + llm.StreamEvent(type="text", chunk="hi", part_index=1), + ] + async_mock_model.enqueue(events) + response = async_mock_model.prompt("x") + chunks = [] + async for chunk in response: + chunks.append(chunk) + assert chunks == ["hi"] + + @pytest.mark.asyncio + async def test_async_messages_requires_await(self, async_mock_model): + async_mock_model.enqueue(["hi"]) + response = async_mock_model.prompt("x") + with pytest.raises(ValueError): + response.messages # noqa: B018 + + @pytest.mark.asyncio + async def test_async_messages_after_await(self, async_mock_model): + async_mock_model.enqueue(["hi"]) + response = async_mock_model.prompt("x") + await response.text() + assert response.messages == [ + llm.Message( + role="assistant", parts=[llm.TextPart(text="hi")] + ) + ] + + +class TestChainResponseStreamEvents: + def test_sync_chain_stream_events_yields_text_when_no_tools( + self, mock_model + ): + # Chain with no tool calls is a single-response chain — its + # stream_events should concatenate from each underlying response. + mock_model.enqueue( + [llm.StreamEvent(type="text", chunk="done", part_index=0)] + ) + chain = mock_model.conversation().chain("q") + events = list(chain.stream_events()) + assert [e.type for e in events] == ["text"] + assert [e.chunk for e in events] == ["done"] + + @pytest.mark.asyncio + async def test_async_chain_astream_events_yields(self, async_mock_model): + async_mock_model.enqueue( + [llm.StreamEvent(type="text", chunk="done", part_index=0)] + ) + chain = async_mock_model.conversation().chain("q") + events = [] + async for event in chain.astream_events(): + events.append(event) + assert [e.type for e in events] == ["text"] From 476d5ef98914bfea2e978494a84ce156e370636e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 20 Apr 2026 09:33:52 -0700 Subject: [PATCH 018/258] Phase 3: messages= parameter and Prompt.messages synthesis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add messages= kwarg to model.prompt(), conversation.prompt(), and their async counterparts. The list flows through to Prompt.__init__ as _explicit_messages. Prompt.messages is now a property that returns one uniform list[Message] regardless of which surface the caller used: - If messages= was passed explicitly, that list is returned verbatim (with any prompt= string appended as a trailing user TextPart, matching how system= sugars into a leading system Message). - Otherwise synthesized from system=, prompt=, attachments=, and tool_results=. Plugins read one representation; callers keep all the existing ergonomic entry points. No plugin changes yet — existing adapters keep reading the legacy prompt.prompt / prompt.system / prompt.attachments. Phase 4 is when the built-in OpenAI plugin switches to reading prompt.messages. 543 tests passing; no regressions. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/models.py | 69 +++++++++++++++++++++ tests/test_parts.py | 145 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+) diff --git a/llm/models.py b/llm/models.py index 003fe0176..6da77b431 100644 --- a/llm/models.py +++ b/llm/models.py @@ -366,6 +366,7 @@ def __init__( schema=None, tools=None, tool_results=None, + messages=None, ): self._prompt = prompt self.model = model @@ -380,6 +381,9 @@ def __init__( self.tools = _wrap_tools(tools or []) self.tool_results = tool_results or [] self.options = options or {} + # Explicit messages= list, if the caller supplied one. Copied so + # later mutation by the caller doesn't alter the Prompt. + self._explicit_messages = list(messages) if messages is not None else None @property def prompt(self): @@ -396,6 +400,63 @@ def system(self): ] return "\n\n".join(bits) + @property + def messages(self): + """Canonical list of Message objects for this prompt. + + If messages= was passed explicitly, returns that list — with the + optional prompt= text appended as a trailing user TextPart. + Otherwise synthesizes from system=, prompt=, attachments=, and + tool_results= so plugins can read one uniform representation + regardless of which surface the caller used. + """ + from .parts import ( + AttachmentPart, + Message, + TextPart, + ToolResultPart, + ) + + if self._explicit_messages is not None: + out = list(self._explicit_messages) + if self._prompt: + out.append( + Message(role="user", parts=[TextPart(text=self._prompt)]) + ) + return out + + result: List["Message"] = [] + + if self.system: + result.append( + Message(role="system", parts=[TextPart(text=self.system)]) + ) + + if self.tool_results: + result.append( + Message( + role="tool", + parts=[ + ToolResultPart( + name=tr.name, + output=tr.output, + tool_call_id=tr.tool_call_id, + ) + for tr in self.tool_results + ], + ) + ) + + user_parts: List[Any] = [] + if self.prompt: + user_parts.append(TextPart(text=self.prompt)) + for att in self.attachments: + user_parts.append(AttachmentPart(attachment=att)) + if user_parts: + result.append(Message(role="user", parts=user_parts)) + + return result + def _wrap_tools(tools: List[ToolDef]) -> List[Tool]: wrapped_tools = [] @@ -442,6 +503,7 @@ def prompt( tools: Optional[List[ToolDef]] = None, tool_results: Optional[List[ToolResult]] = None, system_fragments: Optional[List[Union[str, Fragment]]] = None, + messages: Optional[List[Any]] = None, stream: bool = True, key: Optional[str] = None, **options, @@ -457,6 +519,7 @@ def prompt( tools=tools or self.tools, tool_results=tool_results, system_fragments=system_fragments, + messages=messages, options=self.model.Options(**options), ), self.model, @@ -579,6 +642,7 @@ def prompt( tools: Optional[List[ToolDef]] = None, tool_results: Optional[List[ToolResult]] = None, system_fragments: Optional[List[str]] = None, + messages: Optional[List[Any]] = None, stream: bool = True, key: Optional[str] = None, **options, @@ -594,6 +658,7 @@ def prompt( tools=tools, tool_results=tool_results, system_fragments=system_fragments, + messages=messages, options=self.model.Options(**options), ), self.model, @@ -2115,6 +2180,7 @@ def prompt( attachments: Optional[List[Attachment]] = None, system: Optional[str] = None, system_fragments: Optional[List[Union[str, Fragment]]] = None, + messages: Optional[List[Any]] = None, stream: bool = True, schema: Optional[Union[dict, type[BaseModel]]] = None, tools: Optional[List[ToolDef]] = None, @@ -2133,6 +2199,7 @@ def prompt( tools=tools, tool_results=tool_results, system_fragments=system_fragments, + messages=messages, model=self, options=self.Options(**options), ), @@ -2227,6 +2294,7 @@ def prompt( tools: Optional[List[ToolDef]] = None, tool_results: Optional[List[ToolResult]] = None, system_fragments: Optional[List[Union[str, Fragment]]] = None, + messages: Optional[List[Any]] = None, stream: bool = True, **options, ) -> AsyncResponse: @@ -2242,6 +2310,7 @@ def prompt( tools=tools, tool_results=tool_results, system_fragments=system_fragments, + messages=messages, model=self, options=self.Options(**options), ), diff --git a/tests/test_parts.py b/tests/test_parts.py index 35ccc8f44..6167d7a92 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -618,6 +618,151 @@ async def test_async_messages_after_await(self, async_mock_model): ] +# -- Phase 3: messages= parameter and Prompt.messages synthesis -------- + + +class TestPromptMessagesSynthesis: + """Prompt.messages constructs a Message list from legacy inputs when + messages= wasn't passed explicitly.""" + + def test_empty_prompt_yields_empty_messages(self, mock_model): + from llm.models import Prompt + + p = Prompt(None, model=mock_model) + assert p.messages == [] + + def test_prompt_text_synthesizes_user_message(self, mock_model): + from llm.models import Prompt + + p = Prompt("hi", model=mock_model) + assert p.messages == [ + llm.Message(role="user", parts=[llm.TextPart(text="hi")]) + ] + + def test_system_and_prompt_synthesizes_two_messages(self, mock_model): + from llm.models import Prompt + + p = Prompt("hi", model=mock_model, system="be brief") + assert p.messages == [ + llm.Message(role="system", parts=[llm.TextPart(text="be brief")]), + llm.Message(role="user", parts=[llm.TextPart(text="hi")]), + ] + + def test_attachments_join_user_message(self, mock_model): + from llm.models import Prompt + + att = llm.Attachment(url="http://example.com/a.jpg") + p = Prompt("look", model=mock_model, attachments=[att]) + assert p.messages == [ + llm.Message( + role="user", + parts=[ + llm.TextPart(text="look"), + llm.AttachmentPart(attachment=att), + ], + ) + ] + + def test_tool_results_become_tool_role_message(self, mock_model): + from llm.models import Prompt + from llm import ToolResult + + tr = ToolResult(name="t", output="ok", tool_call_id="c1") + p = Prompt(None, model=mock_model, tool_results=[tr]) + assert p.messages == [ + llm.Message( + role="tool", + parts=[ + llm.ToolResultPart( + name="t", output="ok", tool_call_id="c1" + ) + ], + ) + ] + + +class TestPromptMessagesExplicit: + """When messages= is passed, it's authoritative.""" + + def test_explicit_messages_returned_verbatim(self, mock_model): + from llm.models import Prompt + + explicit = [ + llm.system("x"), + llm.user("y"), + ] + p = Prompt(None, model=mock_model, messages=explicit) + assert p.messages == explicit + + def test_explicit_messages_plus_prompt_appends_trailing_user( + self, mock_model + ): + from llm.models import Prompt + + explicit = [llm.system("x"), llm.user("prior")] + p = Prompt("follow-up", model=mock_model, messages=explicit) + assert p.messages == [ + llm.system("x"), + llm.user("prior"), + llm.user("follow-up"), + ] + + def test_explicit_messages_independent_copy(self, mock_model): + """Mutating the caller's list must not mutate Prompt.messages.""" + from llm.models import Prompt + + explicit = [llm.user("x")] + p = Prompt(None, model=mock_model, messages=explicit) + explicit.append(llm.user("later")) + assert p.messages == [llm.user("x")] + + +class TestModelPromptMessagesKwarg: + """model.prompt / conversation.prompt / async counterparts accept + messages= and the list is observable on the resulting Prompt.""" + + def test_model_prompt_accepts_messages(self, mock_model): + mock_model.enqueue(["ok"]) + response = mock_model.prompt(messages=[llm.user("hi")]) + response.text() + assert response.prompt.messages == [llm.user("hi")] + + def test_model_prompt_messages_with_system(self, mock_model): + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + messages=[llm.system("be brief"), llm.user("hi")] + ) + response.text() + assert response.prompt.messages == [ + llm.system("be brief"), + llm.user("hi"), + ] + + def test_conversation_prompt_accepts_messages(self, mock_model): + mock_model.enqueue(["ok"]) + conv = mock_model.conversation() + response = conv.prompt(messages=[llm.user("q")]) + response.text() + assert response.prompt.messages == [llm.user("q")] + + @pytest.mark.asyncio + async def test_async_model_prompt_accepts_messages(self, async_mock_model): + async_mock_model.enqueue(["ok"]) + response = async_mock_model.prompt(messages=[llm.user("hi")]) + await response.text() + assert response.prompt.messages == [llm.user("hi")] + + @pytest.mark.asyncio + async def test_async_conversation_prompt_accepts_messages( + self, async_mock_model + ): + async_mock_model.enqueue(["ok"]) + conv = async_mock_model.conversation() + response = conv.prompt(messages=[llm.user("q")]) + await response.text() + assert response.prompt.messages == [llm.user("q")] + + class TestChainResponseStreamEvents: def test_sync_chain_stream_events_yields_text_when_no_tools( self, mock_model From 94837936dfaa014dfa3f781f2fc88f0afebd7217 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 20 Apr 2026 09:36:23 -0700 Subject: [PATCH 019/258] Phase 4a: OpenAI build_messages reads prompt.messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite the OpenAI _Shared.build_messages() to consume prompt.messages (the canonical structured input) and dispatch per Part subtype into OpenAI's wire format. Prior-turn input history also comes from prev_response.prompt.messages; prior-turn output continues to use the flat text_or_raise() / tool_calls_or_raise() accessors since those tolerate plugin-specific quirks that _build_parts rejects. Per-Part translation: - TextPart → message content (string) or {"type": "text", ...} entry inside attachment-bearing array content - AttachmentPart → passed through _attachment() for image/audio/pdf - ToolCallPart → tool_calls[] entry on an assistant message (content=null when only tool_calls, no text) - ToolResultPart → one {"role": "tool", tool_call_id, content} per result (Message role="tool" can carry several) System dedup: consecutive identical system prompts emit once. The legacy prompt=/system=/attachments=/tool_results= path continues to work unchanged because Prompt.messages synthesizes those into the same Message list the new code reads. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/default_plugins/openai_models.py | 178 +++++++++++------- tests/test_openai_messages.py | 264 +++++++++++++++++++++++++++ 2 files changed, 376 insertions(+), 66 deletions(-) create mode 100644 tests/test_openai_messages.py diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 55fa7a38d..6d5dcf20c 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -630,81 +630,127 @@ def __init__( def __str__(self) -> str: return "OpenAI Chat: {}".format(self.model_id) + def _append_llm_message(self, out, message, current_system): + """Translate one llm.Message into one (or more) OpenAI message + dicts and append them to ``out``. + + Returns the (possibly updated) current_system value so the caller + can dedup consecutive identical system messages. + """ + from llm.parts import ( + AttachmentPart, + TextPart, + ToolCallPart, + ToolResultPart, + ) + + text_bits = [] + attachment_items = [] + tool_calls = [] + tool_results = [] + + for part in message.parts: + if isinstance(part, TextPart): + text_bits.append(part.text) + elif isinstance(part, AttachmentPart) and part.attachment: + attachment_items.append(_attachment(part.attachment)) + elif isinstance(part, ToolCallPart): + tool_calls.append( + { + "type": "function", + "id": part.tool_call_id, + "function": { + "name": part.name, + "arguments": json.dumps(part.arguments), + }, + } + ) + elif isinstance(part, ToolResultPart): + tool_results.append( + { + "role": "tool", + "tool_call_id": part.tool_call_id, + "content": part.output, + } + ) + + # Role "tool" emits one OpenAI "tool" message per ToolResultPart. + if message.role == "tool": + out.extend(tool_results) + return current_system + + # System dedup — skip if we just emitted this exact system text. + if message.role == "system": + text = "".join(text_bits) + if text == current_system: + return current_system + current_system = text + + if attachment_items: + content = [] + if text_bits: + content.append({"type": "text", "text": "".join(text_bits)}) + content.extend(attachment_items) + entry = {"role": message.role, "content": content} + else: + entry = { + "role": message.role, + "content": "".join(text_bits) if text_bits else None, + } + + if tool_calls: + entry["tool_calls"] = tool_calls + # OpenAI expects content=null when only tool_calls are present. + if not text_bits: + entry["content"] = None + elif entry["content"] is None and message.role != "assistant": + # For user/system, an empty message is pointless — drop it. + return current_system + + out.append(entry) + return current_system + def build_messages(self, prompt, conversation): messages = [] current_system = None if conversation is not None: for prev_response in conversation.responses: - if ( - prev_response.prompt.system - and prev_response.prompt.system != current_system - ): - messages.append( - {"role": "system", "content": prev_response.prompt.system} - ) - current_system = prev_response.prompt.system - if prev_response.attachments: - attachment_message = [] - if prev_response.prompt.prompt: - attachment_message.append( - {"type": "text", "text": prev_response.prompt.prompt} - ) - for attachment in prev_response.attachments: - attachment_message.append(_attachment(attachment)) - messages.append({"role": "user", "content": attachment_message}) - elif prev_response.prompt.prompt: - messages.append( - {"role": "user", "content": prev_response.prompt.prompt} - ) - for tool_result in prev_response.prompt.tool_results: - messages.append( - { - "role": "tool", - "tool_call_id": tool_result.tool_call_id, - "content": tool_result.output, - } + # Input side for the prior turn — read prompt.messages + # so explicit messages= from prior calls round-trips. + for msg in prev_response.prompt.messages: + current_system = self._append_llm_message( + messages, msg, current_system ) + # Output side — use the flat accessors. They tolerate + # the fact that some plugins mix text and tool_calls at + # the same part_index, which _build_parts would reject. prev_text = prev_response.text_or_raise() - if prev_text: - messages.append({"role": "assistant", "content": prev_text}) tool_calls = prev_response.tool_calls_or_raise() - if tool_calls: - messages.append( - { - "role": "assistant", - "tool_calls": [ - { - "type": "function", - "id": tool_call.tool_call_id, - "function": { - "name": tool_call.name, - "arguments": json.dumps(tool_call.arguments), - }, - } - for tool_call in tool_calls - ], - } - ) - if prompt.system and prompt.system != current_system: - messages.append({"role": "system", "content": prompt.system}) - for tool_result in prompt.tool_results: - messages.append( - { - "role": "tool", - "tool_call_id": tool_result.tool_call_id, - "content": tool_result.output, - } + if prev_text or tool_calls: + entry = { + "role": "assistant", + "content": prev_text if prev_text else None, + } + if tool_calls: + entry["tool_calls"] = [ + { + "type": "function", + "id": tc.tool_call_id, + "function": { + "name": tc.name, + "arguments": json.dumps(tc.arguments), + }, + } + for tc in tool_calls + ] + messages.append(entry) + + # Current turn — consume prompt.messages (auto-synthesized from + # legacy kwargs when messages= wasn't explicitly passed). + for msg in prompt.messages: + current_system = self._append_llm_message( + messages, msg, current_system ) - if not prompt.attachments: - if prompt.prompt: - messages.append({"role": "user", "content": prompt.prompt or ""}) - else: - attachment_message = [] - if prompt.prompt: - attachment_message.append({"type": "text", "text": prompt.prompt}) - for attachment in prompt.attachments: - attachment_message.append(_attachment(attachment)) - messages.append({"role": "user", "content": attachment_message}) return messages def set_usage(self, response, usage): diff --git a/tests/test_openai_messages.py b/tests/test_openai_messages.py new file mode 100644 index 000000000..c310e7493 --- /dev/null +++ b/tests/test_openai_messages.py @@ -0,0 +1,264 @@ +"""Tests for the OpenAI built-in plugin's messages= path. + +Phase 4a covers build_messages reading prompt.messages (instead of the +legacy prompt.prompt / prompt.system / prompt.attachments fields), which +lets users pass structured message history via model.prompt(messages=[...]). +""" + +import json + +import pytest + +import llm +from llm.default_plugins.openai_models import Chat +from llm.models import Prompt + + +@pytest.fixture +def chat_model(): + # A plain Chat instance with vision and tools enabled — enough + # capabilities for the Part subtypes we translate. + return Chat("gpt-4o-mini", vision=True, supports_tools=True) + + +class TestBuildMessagesFromExplicitMessages: + def test_single_user_message(self, chat_model): + prompt = Prompt( + None, model=chat_model, messages=[llm.user("hi")] + ) + result = chat_model.build_messages(prompt, None) + assert result == [{"role": "user", "content": "hi"}] + + def test_system_plus_user(self, chat_model): + prompt = Prompt( + None, + model=chat_model, + messages=[llm.system("be brief"), llm.user("hi")], + ) + result = chat_model.build_messages(prompt, None) + assert result == [ + {"role": "system", "content": "be brief"}, + {"role": "user", "content": "hi"}, + ] + + def test_user_with_attachment(self, chat_model): + att = llm.Attachment( + type="image/jpeg", url="http://example.com/cat.jpg" + ) + prompt = Prompt( + None, + model=chat_model, + messages=[llm.user("describe", att)], + ) + result = chat_model.build_messages(prompt, None) + assert result == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe"}, + { + "type": "image_url", + "image_url": {"url": "http://example.com/cat.jpg"}, + }, + ], + } + ] + + def test_assistant_with_tool_call(self, chat_model): + tool_call = llm.ToolCallPart( + name="search", + arguments={"q": "weather"}, + tool_call_id="c1", + ) + prompt = Prompt( + None, + model=chat_model, + messages=[ + llm.user("search weather"), + llm.assistant("on it", tool_call), + ], + ) + result = chat_model.build_messages(prompt, None) + assert result == [ + {"role": "user", "content": "search weather"}, + { + "role": "assistant", + "content": "on it", + "tool_calls": [ + { + "type": "function", + "id": "c1", + "function": { + "name": "search", + "arguments": json.dumps({"q": "weather"}), + }, + } + ], + }, + ] + + def test_assistant_tool_call_only_no_text(self, chat_model): + """When an assistant message has tool_calls but no text, OpenAI + expects content=null.""" + tool_call = llm.ToolCallPart( + name="search", arguments={"q": "x"}, tool_call_id="c1" + ) + prompt = Prompt( + None, + model=chat_model, + messages=[llm.user("q"), llm.assistant(tool_call)], + ) + result = chat_model.build_messages(prompt, None) + assert result[1] == { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "type": "function", + "id": "c1", + "function": { + "name": "search", + "arguments": json.dumps({"q": "x"}), + }, + } + ], + } + + def test_tool_role_message_with_tool_result(self, chat_model): + tr = llm.ToolResultPart( + name="search", output="sunny", tool_call_id="c1" + ) + prompt = Prompt( + None, + model=chat_model, + messages=[ + llm.user("q"), + llm.tool_message(tr), + ], + ) + result = chat_model.build_messages(prompt, None) + assert result == [ + {"role": "user", "content": "q"}, + {"role": "tool", "tool_call_id": "c1", "content": "sunny"}, + ] + + def test_multiple_tool_results_emit_multiple_messages(self, chat_model): + """Parallel tool results: one OpenAI 'tool' message per result.""" + a = llm.ToolResultPart(name="t", output="A", tool_call_id="c1") + b = llm.ToolResultPart(name="t", output="B", tool_call_id="c2") + prompt = Prompt( + None, + model=chat_model, + messages=[llm.user("q"), llm.tool_message(a, b)], + ) + result = chat_model.build_messages(prompt, None) + assert result == [ + {"role": "user", "content": "q"}, + {"role": "tool", "tool_call_id": "c1", "content": "A"}, + {"role": "tool", "tool_call_id": "c2", "content": "B"}, + ] + + +class TestBuildMessagesLegacyFieldsStillWork: + """prompt=, system=, attachments= keep working — they synthesize + messages via Prompt.messages before build_messages sees them.""" + + def test_prompt_only(self, chat_model): + prompt = Prompt("hi", model=chat_model) + result = chat_model.build_messages(prompt, None) + assert result == [{"role": "user", "content": "hi"}] + + def test_system_and_prompt(self, chat_model): + prompt = Prompt("hi", model=chat_model, system="be brief") + result = chat_model.build_messages(prompt, None) + assert result == [ + {"role": "system", "content": "be brief"}, + {"role": "user", "content": "hi"}, + ] + + def test_attachments(self, chat_model): + att = llm.Attachment(type="image/jpeg", url="http://example.com/a.jpg") + prompt = Prompt("look", model=chat_model, attachments=[att]) + result = chat_model.build_messages(prompt, None) + assert result == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + { + "type": "image_url", + "image_url": {"url": "http://example.com/a.jpg"}, + }, + ], + } + ] + + +class TestBuildMessagesSystemDedup: + def test_same_system_not_repeated(self, chat_model): + """If two turns share a system prompt, only the first emits it.""" + # Simulate a conversation with a prior response plus a current + # turn; both have the same system prompt. + from llm import Conversation, Response + + conv = Conversation(model=chat_model) + prev_prompt = Prompt( + "first question", model=chat_model, system="be brief" + ) + prev_response = Response(prev_prompt, chat_model, stream=False) + prev_response._chunks = ["first answer"] + prev_response._done = True + conv.responses = [prev_response] + + new_prompt = Prompt( + "second question", model=chat_model, system="be brief" + ) + + result = chat_model.build_messages(new_prompt, conv) + # System appears once. + system_msgs = [m for m in result if m["role"] == "system"] + assert len(system_msgs) == 1 + assert system_msgs[0]["content"] == "be brief" + + def test_system_change_emitted(self, chat_model): + from llm import Conversation, Response + + conv = Conversation(model=chat_model) + prev_prompt = Prompt( + "q1", model=chat_model, system="be brief" + ) + prev_response = Response(prev_prompt, chat_model, stream=False) + prev_response._chunks = ["a1"] + prev_response._done = True + conv.responses = [prev_response] + + new_prompt = Prompt( + "q2", model=chat_model, system="be expansive" + ) + + result = chat_model.build_messages(new_prompt, conv) + system_msgs = [m for m in result if m["role"] == "system"] + assert [m["content"] for m in system_msgs] == [ + "be brief", + "be expansive", + ] + + +class TestBuildMessagesConversationHistory: + def test_prior_turn_text_plus_current_user(self, chat_model): + from llm import Conversation, Response + + conv = Conversation(model=chat_model) + prev_prompt = Prompt("what's 1+1?", model=chat_model) + prev_response = Response(prev_prompt, chat_model, stream=False) + prev_response._chunks = ["2"] + prev_response._done = True + conv.responses = [prev_response] + + new_prompt = Prompt("what about 2+2?", model=chat_model) + result = chat_model.build_messages(new_prompt, conv) + assert result == [ + {"role": "user", "content": "what's 1+1?"}, + {"role": "assistant", "content": "2"}, + {"role": "user", "content": "what about 2+2?"}, + ] From 8ae96440167ddcfbf79d1d47c937ee700a764d21 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 20 Apr 2026 09:39:00 -0700 Subject: [PATCH 020/258] Phase 4b: OpenAI execute() yields StreamEvent objects Replace string yields with typed StreamEvent yields in both sync Chat.execute and async AsyncChat.execute, for both streaming and non-streaming code paths. Event shape: - Text chunks emit StreamEvent(type="text", part_index=0). Empty-string content (OpenAI's first role=assistant delta) is now skipped as noise. - Each tool call gets its own part_index past any text that preceded it (text is always part_index=0, tool calls start at 1). A new tool call emits tool_call_name with name+id, and each tool-call-args delta emits tool_call_args with the partial JSON. Callers see arguments build up live via stream_events(). - Non-streaming path emits one StreamEvent per content block: tool calls first (each with name + args events), then the final text if present. response.add_tool_call() is still invoked for every tool call so existing code paths (response.tool_calls(), chain execution) keep working. The new event stream is additive. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/default_plugins/openai_models.py | 124 +++++++++--- tests/test_openai_messages.py | 272 +++++++++++++++++++++++++++ 2 files changed, 372 insertions(+), 24 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 6d5dcf20c..f8f09e453 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -7,6 +7,7 @@ KeyModel, Prompt, Response, + StreamEvent, hookimpl, ) import llm @@ -851,6 +852,12 @@ def execute( ) chunks = [] tool_calls = {} + # part_index allocator. Text always uses 0. Each tool call + # at delta index i is assigned a part_index past any text + # that was seen, so _build_parts groups them correctly. + seen_text = False + tc_part_index = {} + next_part_index = 1 for chunk in completion: chunks.append(chunk) if chunk.usage: @@ -859,24 +866,42 @@ def execute( for tool_call in chunk.choices[0].delta.tool_calls or []: if tool_call.function.arguments is None: tool_call.function.arguments = "" - index = tool_call.index - if index not in tool_calls: - tool_calls[index] = tool_call + idx = tool_call.index + if idx not in tool_calls: + tool_calls[idx] = tool_call + tc_part_index[idx] = next_part_index + next_part_index += 1 + yield StreamEvent( + type="tool_call_name", + chunk=tool_call.function.name or "", + part_index=tc_part_index[idx], + tool_call_id=tool_call.id, + ) else: - tool_calls[ - index - ].function.arguments += tool_call.function.arguments + tool_calls[idx].function.arguments += ( + tool_call.function.arguments + ) + if tool_call.function.arguments: + yield StreamEvent( + type="tool_call_args", + chunk=tool_call.function.arguments, + part_index=tc_part_index[idx], + tool_call_id=tool_calls[idx].id, + ) try: content = chunk.choices[0].delta.content except IndexError: content = None - if content is not None: - yield content + if content: + # Empty strings are noise (OpenAI's first chunk + # with role=assistant has content=""). + seen_text = True + yield StreamEvent( + type="text", chunk=content, part_index=0 + ) response.response_json = remove_dict_none_values(combine_chunks(chunks)) if tool_calls: for value in tool_calls.values(): - # value.function looks like this: - # ChoiceDeltaToolCallFunction(arguments='{"city":"San Francisco"}', name='get_weather') response.add_tool_call( llm.ToolCall( tool_call_id=value.id, @@ -893,6 +918,7 @@ def execute( ) usage = completion.usage.model_dump() response.response_json = remove_dict_none_values(completion.model_dump()) + part_index = 0 for tool_call in completion.choices[0].message.tool_calls or []: response.add_tool_call( llm.ToolCall( @@ -901,8 +927,25 @@ def execute( arguments=json.loads(tool_call.function.arguments), ) ) + part_index += 1 + yield StreamEvent( + type="tool_call_name", + chunk=tool_call.function.name or "", + part_index=part_index, + tool_call_id=tool_call.id, + ) + yield StreamEvent( + type="tool_call_args", + chunk=tool_call.function.arguments or "", + part_index=part_index, + tool_call_id=tool_call.id, + ) if completion.choices[0].message.content is not None: - yield completion.choices[0].message.content + yield StreamEvent( + type="text", + chunk=completion.choices[0].message.content, + part_index=0, + ) self.set_usage(response, usage) response._prompt_json = redact_data({"messages": messages}) @@ -941,33 +984,48 @@ async def execute( ) chunks = [] tool_calls = {} + tc_part_index = {} + next_part_index = 1 async for chunk in completion: if chunk.usage: usage = chunk.usage.model_dump() chunks.append(chunk) - if chunk.usage: - usage = chunk.usage.model_dump() if chunk.choices and chunk.choices[0].delta: for tool_call in chunk.choices[0].delta.tool_calls or []: if tool_call.function.arguments is None: tool_call.function.arguments = "" - index = tool_call.index - if index not in tool_calls: - tool_calls[index] = tool_call + idx = tool_call.index + if idx not in tool_calls: + tool_calls[idx] = tool_call + tc_part_index[idx] = next_part_index + next_part_index += 1 + yield StreamEvent( + type="tool_call_name", + chunk=tool_call.function.name or "", + part_index=tc_part_index[idx], + tool_call_id=tool_call.id, + ) else: - tool_calls[ - index - ].function.arguments += tool_call.function.arguments + tool_calls[idx].function.arguments += ( + tool_call.function.arguments + ) + if tool_call.function.arguments: + yield StreamEvent( + type="tool_call_args", + chunk=tool_call.function.arguments, + part_index=tc_part_index[idx], + tool_call_id=tool_calls[idx].id, + ) try: content = chunk.choices[0].delta.content except IndexError: content = None - if content is not None: - yield content + if content: + yield StreamEvent( + type="text", chunk=content, part_index=0 + ) if tool_calls: for value in tool_calls.values(): - # value.function looks like this: - # ChoiceDeltaToolCallFunction(arguments='{"city":"San Francisco"}', name='get_weather') response.add_tool_call( llm.ToolCall( tool_call_id=value.id, @@ -985,6 +1043,7 @@ async def execute( ) response.response_json = remove_dict_none_values(completion.model_dump()) usage = completion.usage.model_dump() + part_index = 0 for tool_call in completion.choices[0].message.tool_calls or []: response.add_tool_call( llm.ToolCall( @@ -993,8 +1052,25 @@ async def execute( arguments=json.loads(tool_call.function.arguments), ) ) + part_index += 1 + yield StreamEvent( + type="tool_call_name", + chunk=tool_call.function.name or "", + part_index=part_index, + tool_call_id=tool_call.id, + ) + yield StreamEvent( + type="tool_call_args", + chunk=tool_call.function.arguments or "", + part_index=part_index, + tool_call_id=tool_call.id, + ) if completion.choices[0].message.content is not None: - yield completion.choices[0].message.content + yield StreamEvent( + type="text", + chunk=completion.choices[0].message.content, + part_index=0, + ) self.set_usage(response, usage) response._prompt_json = redact_data({"messages": messages}) diff --git a/tests/test_openai_messages.py b/tests/test_openai_messages.py index c310e7493..c398941f1 100644 --- a/tests/test_openai_messages.py +++ b/tests/test_openai_messages.py @@ -3,17 +3,105 @@ Phase 4a covers build_messages reading prompt.messages (instead of the legacy prompt.prompt / prompt.system / prompt.attachments fields), which lets users pass structured message history via model.prompt(messages=[...]). + +Phase 4b covers execute() yielding StreamEvent objects instead of plain +str — including tool_call_name + tool_call_args event streams. """ import json import pytest +from pytest_httpx import IteratorStream import llm from llm.default_plugins.openai_models import Chat from llm.models import Prompt +API_KEY = "badkey" + + +def _sse(delta, finish_reason=None, usage=None, tool_calls=None): + chunk = { + "id": "c1", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "delta": delta, "finish_reason": finish_reason} + ], + } + if tool_calls is not None: + chunk["choices"][0]["delta"]["tool_calls"] = tool_calls + if usage is not None: + chunk["usage"] = usage + return f"data: {json.dumps(chunk)}\n\n".encode("utf-8") + + +def _text_stream(): + yield _sse({"role": "assistant", "content": ""}) + yield _sse({"content": "Hel"}) + yield _sse({"content": "lo"}) + yield _sse({}, finish_reason="stop") + yield b"data: [DONE]\n\n" + + +def _tool_call_stream(): + """Mimic an OpenAI stream with a tool call (no preceding text).""" + yield _sse({"role": "assistant", "content": None}) + yield _sse( + {}, + tool_calls=[ + { + "index": 0, + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": ""}, + } + ], + ) + yield _sse( + {}, + tool_calls=[ + { + "index": 0, + "function": {"arguments": '{"city":'}, + } + ], + ) + yield _sse( + {}, + tool_calls=[ + { + "index": 0, + "function": {"arguments": '"Paris"}'}, + } + ], + ) + yield _sse({}, finish_reason="tool_calls") + yield b"data: [DONE]\n\n" + + +def _text_then_tool_call_stream(): + """Text arrives first, then a tool call — the tool call must get + a part_index past the text so assembly doesn't mix families.""" + yield _sse({"role": "assistant", "content": ""}) + yield _sse({"content": "Looking up"}) + yield _sse( + {}, + tool_calls=[ + { + "index": 0, + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"c":1}'}, + } + ], + ) + yield _sse({}, finish_reason="tool_calls") + yield b"data: [DONE]\n\n" + + @pytest.fixture def chat_model(): # A plain Chat instance with vision and tools enabled — enough @@ -262,3 +350,187 @@ def test_prior_turn_text_plus_current_user(self, chat_model): {"role": "assistant", "content": "2"}, {"role": "user", "content": "what about 2+2?"}, ] + + +# -- Phase 4b: execute() yields StreamEvents --------------------------- + + +class TestStreamingExecuteYieldsStreamEvents: + def test_text_stream_yields_text_events(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_text_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + model = llm.get_model("gpt-4o-mini") + response = model.prompt("hi", key=API_KEY) + events = list(response.stream_events()) + # At least one StreamEvent, all text, all at part_index=0. + assert events, "expected stream events" + assert all(isinstance(e, llm.StreamEvent) for e in events) + assert all(e.type == "text" for e in events) + assert all(e.part_index == 0 for e in events) + # Text chunks concatenate to the expected full text. + assert "".join(e.chunk for e in events) == "Hello" + + def test_text_stream_plain_iteration_still_returns_strings( + self, httpx_mock + ): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_text_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + model = llm.get_model("gpt-4o-mini") + response = model.prompt("hi", key=API_KEY) + chunks = list(response) + assert all(isinstance(c, str) for c in chunks) + assert "".join(chunks) == "Hello" + + def test_text_stream_messages_assembled(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_text_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + model = llm.get_model("gpt-4o-mini") + response = model.prompt("hi", key=API_KEY) + response.text() + assert response.messages == [ + llm.Message( + role="assistant", parts=[llm.TextPart(text="Hello")] + ) + ] + + def test_tool_call_stream_yields_name_and_args_events(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_tool_call_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + + def get_weather(city: str) -> str: + "Look up the weather." + return "sunny" + + model = llm.get_model("gpt-4o-mini") + response = model.prompt("weather?", tools=[get_weather], key=API_KEY) + events = list(response.stream_events()) + types = [e.type for e in events] + assert "tool_call_name" in types + assert "tool_call_args" in types + # Name event carries the tool_call_id and name. + name_ev = next(e for e in events if e.type == "tool_call_name") + assert name_ev.tool_call_id == "call_1" + assert name_ev.chunk == "get_weather" + # Args events share the same part_index and concatenate to + # valid JSON. + args_events = [e for e in events if e.type == "tool_call_args"] + assert all(e.part_index == name_ev.part_index for e in args_events) + assert json.loads("".join(e.chunk for e in args_events)) == { + "city": "Paris" + } + + def test_tool_call_registered_via_add_tool_call(self, httpx_mock): + """response.tool_calls() still works — chain/execute relies on it.""" + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_tool_call_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + + def get_weather(city: str) -> str: + "Look up the weather." + return "sunny" + + model = llm.get_model("gpt-4o-mini") + response = model.prompt("weather?", tools=[get_weather], key=API_KEY) + response.text() + tcs = response.tool_calls() + assert len(tcs) == 1 + assert tcs[0].name == "get_weather" + assert tcs[0].arguments == {"city": "Paris"} + assert tcs[0].tool_call_id == "call_1" + + def test_text_then_tool_call_part_index_advances(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_text_then_tool_call_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + + def get_weather(c: int) -> str: + "Weather." + return "sunny" + + model = llm.get_model("gpt-4o-mini") + response = model.prompt("q", tools=[get_weather], key=API_KEY) + response.text() + # After streaming, messages has both a TextPart and a ToolCallPart. + parts = response.messages[0].parts + assert any(isinstance(p, llm.TextPart) for p in parts) + assert any(isinstance(p, llm.ToolCallPart) for p in parts) + text_part = next(p for p in parts if isinstance(p, llm.TextPart)) + tc_part = next(p for p in parts if isinstance(p, llm.ToolCallPart)) + assert text_part.text == "Looking up" + assert tc_part.name == "get_weather" + assert tc_part.arguments == {"c": 1} + + +class TestAsyncStreamingExecuteYieldsStreamEvents: + @pytest.mark.asyncio + async def test_text_stream_yields_text_events(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_text_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + model = llm.get_async_model("gpt-4o-mini") + response = model.prompt("hi", key=API_KEY) + events = [] + async for event in response.astream_events(): + events.append(event) + assert all(isinstance(e, llm.StreamEvent) for e in events) + assert [e.type for e in events] == ["text"] * len(events) + assert "".join(e.chunk for e in events) == "Hello" + + +class TestNonStreamingExecuteYieldsStreamEvents: + def test_non_streaming_text_yields_single_event(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "model": "gpt-4o-mini", + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + "choices": [ + { + "message": {"role": "assistant", "content": "Hello"}, + "finish_reason": "stop", + } + ], + }, + headers={"Content-Type": "application/json"}, + ) + model = llm.get_model("gpt-4o-mini") + response = model.prompt("hi", key=API_KEY, stream=False) + events = list(response.stream_events()) + assert events == [ + llm.StreamEvent(type="text", chunk="Hello", part_index=0) + ] + assert response.messages == [ + llm.Message( + role="assistant", parts=[llm.TextPart(text="Hello")] + ) + ] From 9f0d72ac03cd995a190da209806dfbc8c80049fc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 20 Apr 2026 09:40:28 -0700 Subject: [PATCH 021/258] =?UTF-8?q?Phase=204c:=20OpenAI=20reasoning=20toke?= =?UTF-8?q?n=20count=20=E2=86=92=20redacted=20ReasoningPart?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capture usage.completion_tokens_details.reasoning_tokens and store it on response._reasoning_token_count BEFORE set_usage runs — set_usage pops top-level keys and then simplify_usage_dict strips zero-valued entries, both of which would lose the count. Phase 2's _build_parts() picks up _reasoning_token_count and prepends a ReasoningPart(redacted=True, token_count=N, text="") to the assembled output messages. That gives CLI and client code a hook to render "GPT-5 used N reasoning tokens here" without the model actually streaming any reasoning text (which OpenAI's reasoning models don't expose). Both Chat.execute (sync) and AsyncChat.execute do this. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/default_plugins/openai_models.py | 20 ++++++++ tests/test_openai_messages.py | 75 ++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index f8f09e453..7549cd0ae 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -946,6 +946,17 @@ def execute( chunk=completion.choices[0].message.content, part_index=0, ) + # Capture the reasoning token count BEFORE set_usage runs — + # set_usage pops top-level keys and passes the rest through + # simplify_usage_dict, which strips zero-valued entries. + if usage: + reasoning_tokens = ( + (usage.get("completion_tokens_details") or {}).get( + "reasoning_tokens", 0 + ) + ) + if reasoning_tokens: + response._reasoning_token_count = reasoning_tokens self.set_usage(response, usage) response._prompt_json = redact_data({"messages": messages}) @@ -1071,6 +1082,15 @@ async def execute( chunk=completion.choices[0].message.content, part_index=0, ) + # See sync Chat.execute: capture reasoning before set_usage mutates. + if usage: + reasoning_tokens = ( + (usage.get("completion_tokens_details") or {}).get( + "reasoning_tokens", 0 + ) + ) + if reasoning_tokens: + response._reasoning_token_count = reasoning_tokens self.set_usage(response, usage) response._prompt_json = redact_data({"messages": messages}) diff --git a/tests/test_openai_messages.py b/tests/test_openai_messages.py index c398941f1..42fff7e14 100644 --- a/tests/test_openai_messages.py +++ b/tests/test_openai_messages.py @@ -502,6 +502,81 @@ async def test_text_stream_yields_text_events(self, httpx_mock): assert "".join(e.chunk for e in events) == "Hello" +def _text_stream_with_reasoning_usage(reasoning_tokens): + """Stream with usage in the final chunk reporting reasoning_tokens.""" + yield _sse({"role": "assistant", "content": ""}) + yield _sse({"content": "Hel"}) + yield _sse({"content": "lo"}) + yield _sse({}, finish_reason="stop") + # Final chunk with usage — OpenAI streams usage once at the end + # when stream_options.include_usage=True. + yield _sse( + {}, + usage={ + "prompt_tokens": 5, + "completion_tokens": 2, + "total_tokens": 7, + "completion_tokens_details": { + "reasoning_tokens": reasoning_tokens + }, + }, + ) + yield b"data: [DONE]\n\n" + + +class TestReasoningTokenCount: + def test_reasoning_token_count_recorded(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_text_stream_with_reasoning_usage(200)), + headers={"Content-Type": "text/event-stream"}, + ) + model = llm.get_model("gpt-4o-mini") + response = model.prompt("hi", key=API_KEY) + response.text() + assert response._reasoning_token_count == 200 + + def test_reasoning_part_prepended_to_messages(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_text_stream_with_reasoning_usage(150)), + headers={"Content-Type": "text/event-stream"}, + ) + model = llm.get_model("gpt-4o-mini") + response = model.prompt("hi", key=API_KEY) + response.text() + assert response.messages == [ + llm.Message( + role="assistant", + parts=[ + llm.ReasoningPart( + text="", redacted=True, token_count=150 + ), + llm.TextPart(text="Hello"), + ], + ) + ] + + def test_no_reasoning_part_when_zero_or_absent(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_text_stream_with_reasoning_usage(0)), + headers={"Content-Type": "text/event-stream"}, + ) + model = llm.get_model("gpt-4o-mini") + response = model.prompt("hi", key=API_KEY) + response.text() + # Either the attribute isn't set, or it's 0 — either way no + # redacted ReasoningPart in the assembled messages. + parts = response.messages[0].parts + assert not any( + isinstance(p, llm.ReasoningPart) for p in parts + ), "should not add a redacted reasoning part when count=0" + + class TestNonStreamingExecuteYieldsStreamEvents: def test_non_streaming_text_yields_single_event(self, httpx_mock): httpx_mock.add_response( From c629c133b733c3c18a53ef3760d0e328b4a43c62 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 20 Apr 2026 09:44:17 -0700 Subject: [PATCH 022/258] Phase 5: CLI reasoning display MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a display_stream_events() helper (plus async twin) that writes text events to stdout and reasoning events to stderr in dim style, with a newline inserted at reasoning→text transitions so the assistant's final answer starts on a fresh visual line. Add -R / --no-reasoning flag to both `llm prompt` and `llm chat` to suppress the stderr reasoning stream. The sync and async streaming paths in `llm prompt` and the streaming path in `llm chat` now consume response.stream_events() / astream_events() through the helper instead of plain iteration. Conftest: MockModel and AsyncMockModel now set can_stream = True so CLI tests exercising event-level behavior actually take the streaming branch. This matches the fixtures' practical behavior — they yield chunks, one at a time. With only the built-in OpenAI plugin upgraded (Phase 4), no model currently streams live reasoning text (OpenAI's reasoning is redacted — just a token count). The machinery is ready for llm-anthropic / llm-gemini upgrades to light up the experience. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/cli.py | 62 ++++++++++++-- tests/conftest.py | 2 + tests/test_cli_streaming.py | 158 ++++++++++++++++++++++++++++++++++++ 3 files changed, 213 insertions(+), 9 deletions(-) create mode 100644 tests/test_cli_streaming.py diff --git a/llm/cli.py b/llm/cli.py index c78c0be6a..f5cdd586d 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -89,6 +89,39 @@ class FragmentNotFound(Exception): pass +def display_stream_events(events, *, show_reasoning=True): + """Consume a sync iterator of StreamEvents and write them. + + Text events go to stdout. Reasoning events go to stderr in dim style. + A newline is written to stderr at each reasoning→text transition so + the assistant text starts on a fresh visual line. + """ + was_reasoning = False + for event in events: + if event.type == "text": + if was_reasoning and show_reasoning: + click.echo("", err=True) + was_reasoning = False + click.echo(event.chunk, nl=False) + elif event.type == "reasoning" and show_reasoning: + was_reasoning = True + click.echo(click.style(event.chunk, dim=True), nl=False, err=True) + + +async def display_async_stream_events(events, *, show_reasoning=True): + """Async counterpart of display_stream_events.""" + was_reasoning = False + async for event in events: + if event.type == "text": + if was_reasoning and show_reasoning: + click.echo("", err=True) + was_reasoning = False + click.echo(event.chunk, nl=False) + elif event.type == "reasoning" and show_reasoning: + was_reasoning = True + click.echo(click.style(event.chunk, dim=True), nl=False, err=True) + + def validate_fragment_alias(ctx, param, value): if not re.match(r"^[a-zA-Z0-9_-]+$", value): raise click.BadParameter("Fragment alias must be alphanumeric") @@ -450,6 +483,9 @@ def cli(): @click.option("--no-stream", is_flag=True, help="Do not stream output") @click.option("-n", "--no-log", is_flag=True, help="Don't log to database") @click.option("--log", is_flag=True, help="Log prompt and response to the database") +@click.option( + "-R", "--no-reasoning", is_flag=True, help="Don't display reasoning output" +) @click.option( "_continue", "-c", @@ -499,6 +535,7 @@ def prompt( no_stream, no_log, log, + no_reasoning, _continue, conversation_id, key, @@ -849,9 +886,10 @@ async def inner(): system_fragments=resolved_system_fragments, **kwargs, ) - async for chunk in response: - print(chunk, end="") - sys.stdout.flush() + await display_async_stream_events( + response.astream_events(), + show_reasoning=not no_reasoning, + ) print("") else: response = prompt_method( @@ -883,9 +921,10 @@ async def inner(): **kwargs, ) if should_stream: - for chunk in response: - print(chunk, end="") - sys.stdout.flush() + display_stream_events( + response.stream_events(), + show_reasoning=not no_reasoning, + ) print("") else: text = response.text() @@ -982,6 +1021,9 @@ async def inner(): help="Path to log database", ) @click.option("--no-stream", is_flag=True, help="Do not stream output") +@click.option( + "-R", "--no-reasoning", is_flag=True, help="Don't display reasoning output" +) @click.option("--key", help="API key to use") @click.option( "tools", @@ -1030,6 +1072,7 @@ def chat( param, options, no_stream, + no_reasoning, key, database, tools, @@ -1234,9 +1277,10 @@ def chat( # System prompt and system fragments only sent for the first message system = None argument_system_fragments = [] - for chunk in response: - print(chunk, end="") - sys.stdout.flush() + display_stream_events( + response.stream_events(), + show_reasoning=not no_reasoning, + ) response.log_to_db(db) print("") diff --git a/tests/conftest.py b/tests/conftest.py index 8b9a7f85c..f004ed095 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -51,6 +51,7 @@ def env_setup(monkeypatch, user_path): class MockModel(llm.Model): model_id = "mock" attachment_types = {"image/png", "audio/wav"} + can_stream = True supports_schema = True supports_tools = True @@ -105,6 +106,7 @@ async def execute(self, prompt, stream, response, conversation, key): class AsyncMockModel(llm.AsyncModel): model_id = "mock" + can_stream = True supports_schema = True def __init__(self): diff --git a/tests/test_cli_streaming.py b/tests/test_cli_streaming.py new file mode 100644 index 000000000..f631f1d91 --- /dev/null +++ b/tests/test_cli_streaming.py @@ -0,0 +1,158 @@ +"""Tests for CLI streaming display: reasoning → stderr (dim), +text → stdout, -R / --no-reasoning flag. +""" + +import click +from click.testing import CliRunner + +import llm +from llm.cli import cli + + +def test_text_goes_to_stdout_not_stderr(mock_model): + mock_model.enqueue(["Hello world"]) + runner = CliRunner(mix_stderr=False) + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--no-log"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert "Hello world" in result.stdout + # No reasoning was emitted — stderr should be empty. + assert result.stderr == "" + + +def test_reasoning_goes_to_stderr_not_stdout(mock_model): + mock_model.enqueue( + [ + llm.StreamEvent( + type="reasoning", chunk="thinking hard", part_index=0 + ), + llm.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) + runner = CliRunner(mix_stderr=False) + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--no-log"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert "thinking hard" in result.stderr + assert "thinking hard" not in result.stdout + assert "answer" in result.stdout + + +def test_reasoning_rendered_in_dim_style(mock_model): + """The click.style(..., dim=True) wrapper emits the ANSI dim code.""" + mock_model.enqueue( + [ + llm.StreamEvent(type="reasoning", chunk="t", part_index=0), + llm.StreamEvent(type="text", chunk="x", part_index=1), + ] + ) + runner = CliRunner(mix_stderr=False) + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--no-log"], + catch_exceptions=False, + color=True, + ) + assert result.exit_code == 0 + # ANSI dim escape sequence is \x1b[2m + dim_start = click.style("x", dim=True).split("x", 1)[0] + assert dim_start in result.stderr + + +def test_no_reasoning_flag_suppresses_reasoning(mock_model): + mock_model.enqueue( + [ + llm.StreamEvent( + type="reasoning", chunk="hidden thinking", part_index=0 + ), + llm.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) + runner = CliRunner(mix_stderr=False) + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--no-log", "--no-reasoning"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert "hidden thinking" not in result.stderr + assert "hidden thinking" not in result.stdout + assert "answer" in result.stdout + + +def test_no_reasoning_short_flag_R(mock_model): + mock_model.enqueue( + [ + llm.StreamEvent(type="reasoning", chunk="hidden", part_index=0), + llm.StreamEvent(type="text", chunk="x", part_index=1), + ] + ) + runner = CliRunner(mix_stderr=False) + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--no-log", "-R"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert "hidden" not in result.stderr + + +def test_newline_between_reasoning_and_text(mock_model): + """When reasoning ends and text begins, stderr gets a newline so the + text on stdout starts on a fresh visual line.""" + mock_model.enqueue( + [ + llm.StreamEvent(type="reasoning", chunk="think", part_index=0), + llm.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) + runner = CliRunner(mix_stderr=False) + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--no-log"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + # Reasoning ends, then a newline is emitted on stderr. + assert result.stderr.rstrip("\n").endswith("think") or "think\n" in result.stderr + + +def test_async_path_reasoning_to_stderr(async_mock_model): + async_mock_model.enqueue( + [ + llm.StreamEvent( + type="reasoning", chunk="async thinking", part_index=0 + ), + llm.StreamEvent(type="text", chunk="async answer", part_index=1), + ] + ) + runner = CliRunner(mix_stderr=False) + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--async", "--no-log"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert "async thinking" in result.stderr + assert "async answer" in result.stdout + + +def test_plain_str_plugin_still_works(mock_model): + """A plugin that yields plain strings (legacy) still displays + correctly — no reasoning branch, everything to stdout.""" + mock_model.enqueue(["plain ", "text"]) + runner = CliRunner(mix_stderr=False) + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--no-log"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert "plain text" in result.stdout + assert result.stderr == "" From 063564d34d864074fc0864c5292e5855dd98490a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 20 Apr 2026 09:45:50 -0700 Subject: [PATCH 023/258] Phase 6: client-side serialization round-trip test + docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lock in the "application does its own persistence without SQLite" story with: - Five integration tests covering: Message.to_dict / from_dict round-trip, re-inflating messages and continuing a conversation, tool calls + results round-trip, redacted reasoning Parts round-trip, and provider_metadata round-trip. - A new "Structured messages and streaming events" section in docs/python-api.md walking users through messages=[...], stream_events(), response.messages, and the JSON round-trip pattern. No new code — the machinery landed in Phases 1-3. This phase is validation + documentation. 580 tests passing overall. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/python-api.md | 107 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_parts.py | 102 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+) diff --git a/docs/python-api.md b/docs/python-api.md index 202d88dc1..117ba6c36 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -526,6 +526,113 @@ If a response has been evaluated, `response.text()` will continue to return the :exclude-members: fake, from_row, log_to_db ``` +(python-api-messages)= + +### Structured messages and streaming events + +LLM has a structured view of a conversation that sits alongside the +simple string API. Each turn is an `llm.Message` with a `role` +(`"user"`, `"assistant"`, `"system"`, or `"tool"`) and a list of +`Part` objects — `TextPart`, `ReasoningPart`, `ToolCallPart`, +`ToolResultPart`, or `AttachmentPart`. You can pass explicit +structured input via `messages=[...]`, observe typed events as the +model streams, and inspect the assembled message after the response +completes. + +```python +import llm +from llm import user, assistant, system + +model = llm.get_model("gpt-4o-mini") + +response = model.prompt(messages=[ + system("You are a helpful pirate."), + user("What is the capital of France?"), + assistant("Paris, matey."), + user("And Germany?"), +]) +print(response.text()) +``` + +The `user`, `assistant`, `system`, and `tool_message` helpers accept +strings (wrapped as `TextPart`), `llm.Attachment` instances (wrapped +as `AttachmentPart`), existing `Part` objects, and nested lists or +tuples. + +The simple `model.prompt("hi", system="Be brief.")` form keeps +working — it's equivalent to +`model.prompt(messages=[system("Be brief."), user("hi")])`. + +#### Streaming events as they arrive + +`response.stream_events()` yields typed events for every content +block the model produces, live: + +```python +response = model.prompt("Explain quantum computing briefly.") +for event in response.stream_events(): + if event.type == "reasoning": + print(f"[thinking] {event.chunk}", end="", flush=True) + elif event.type == "text": + print(event.chunk, end="", flush=True) + elif event.type == "tool_call_name": + print(f"\n[calling tool: {event.chunk}]") + elif event.type == "tool_call_args": + print(event.chunk, end="", flush=True) +``` + +Event types are `"text"`, `"reasoning"`, `"tool_call_name"`, +`"tool_call_args"`, and `"tool_result"`. Each event carries a +`part_index` that groups events into the same logical Part (all +events at the same `part_index` assemble into one Part after the +stream completes). For async models, use +`async for event in response.astream_events()`. + +Plain iteration (`for chunk in response`) continues to yield only +text strings — reasoning and tool-call events are filtered out. + +#### Inspecting the finished response + +After a response completes, `response.messages` gives you the +assembled list of `Message` objects: + +```python +response = model.prompt("What's 2+2?") +response.text() +for message in response.messages: + for part in message.parts: + print(type(part).__name__, part.to_dict()) +``` + +#### Persisting a conversation yourself + +Messages and Parts round-trip through plain Python dicts via +`to_dict()` / `from_dict()`, so your application can persist +conversations to any JSON-capable store without touching SQLite: + +```python +import json + +# Turn 1 +response = model.prompt("What's 2+2?") +response.text() + +# Build a history payload from the user prompt + assistant reply. +history = [user("What's 2+2?").to_dict()] + [ + m.to_dict() for m in response.messages +] +payload = json.dumps(history) +# ...save `payload` wherever you want... + +# Later — re-inflate and continue. +rebuilt = [llm.Message.from_dict(d) for d in json.loads(payload)] +response = model.prompt(messages=rebuilt + [user("And 3+3?")]) +print(response.text()) +``` + +`AttachmentPart` bytes are base64-encoded in the dict form, so full +multi-modal conversations round-trip faithfully. + (python-api-async)= ## Async models diff --git a/tests/test_parts.py b/tests/test_parts.py index 6167d7a92..17cab2027 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -787,3 +787,105 @@ async def test_async_chain_astream_events_yields(self, async_mock_model): async for event in chain.astream_events(): events.append(event) assert [e.type for e in events] == ["text"] + + +# -- Phase 6: Client-side serialization round-trip --------------------- +# +# A library user can persist a conversation by serializing response.messages +# to JSON and later re-inflate it as messages=[...] on a follow-up prompt. +# No SQLite involvement. + + +class TestClientSerializationRoundTrip: + def test_response_messages_json_roundtrip(self, mock_model): + mock_model.enqueue(["hello there"]) + r = mock_model.prompt("hi") + r.text() + + # Serialize via Message.to_dict / json.dumps + payload = json.dumps([m.to_dict() for m in r.messages]) + # Deserialize — no LLM state needed beyond the types. + restored = [llm.Message.from_dict(d) for d in json.loads(payload)] + + assert restored == r.messages + + def test_rebuilt_messages_reach_plugin_via_prompt(self, mock_model): + """Round-trip: serialize messages from turn 1, re-inflate, send + as messages= to turn 2. The plugin sees the full chain.""" + # Turn 1 + mock_model.enqueue(["turn 1 answer"]) + r1 = mock_model.prompt("turn 1 question") + r1.text() + + # Persist everything the client cares about. + history = [llm.user("turn 1 question").to_dict()] + [ + m.to_dict() for m in r1.messages + ] + payload = json.dumps(history) + + # Later — rebuild from the wire form and continue. + rebuilt = [llm.Message.from_dict(d) for d in json.loads(payload)] + mock_model.enqueue(["turn 2 answer"]) + r2 = mock_model.prompt( + messages=rebuilt + [llm.user("turn 2 question")] + ) + r2.text() + + # The plugin saw the full structured history on prompt.messages. + assert r2.prompt.messages == rebuilt + [ + llm.user("turn 2 question") + ] + assert r2.messages == [llm.assistant("turn 2 answer")] + + def test_roundtrip_preserves_tool_calls_and_results(self, mock_model): + """Assistant messages with tool calls + subsequent tool role + messages survive json round-trip intact.""" + messages = [ + llm.user("what's the weather?"), + llm.assistant( + "let me check", + llm.ToolCallPart( + name="get_weather", + arguments={"city": "Paris"}, + tool_call_id="c1", + ), + ), + llm.tool_message( + llm.ToolResultPart( + name="get_weather", + output="sunny", + tool_call_id="c1", + ) + ), + ] + payload = json.dumps([m.to_dict() for m in messages]) + restored = [llm.Message.from_dict(d) for d in json.loads(payload)] + assert restored == messages + + def test_roundtrip_preserves_redacted_reasoning(self, mock_model): + """Redacted reasoning parts (opaque token counts) survive + round-trip — needed for accurate rendering of 'this turn used + N reasoning tokens'.""" + msg = llm.Message( + role="assistant", + parts=[ + llm.ReasoningPart(text="", redacted=True, token_count=150), + llm.TextPart(text="result"), + ], + ) + restored = llm.Message.from_dict(json.loads(json.dumps(msg.to_dict()))) + assert restored == msg + + def test_roundtrip_preserves_provider_metadata(self, mock_model): + msg = llm.Message( + role="assistant", + parts=[ + llm.ReasoningPart( + text="thinking", + provider_metadata={"anthropic": {"signature": "abc"}}, + ), + llm.TextPart(text="answer"), + ], + ) + restored = llm.Message.from_dict(json.loads(json.dumps(msg.to_dict()))) + assert restored == msg From 63cdf94fbe13651ebdd4483cf14ca9d5c66a0e30 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Apr 2026 08:17:31 -0700 Subject: [PATCH 024/258] Phase 7: to_dict/from_dict/reply + full-chain invariant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three new Response primitives that make conversation persistence outside SQLite ergonomic — and a clarified invariant that response.prompt.messages is always exactly what the model was sent. response.reply(text, **kwargs) -> Response Builds next-turn chain = self.prompt.messages + self.messages + [user(text)] and calls self.model.prompt(messages=chain, ...). Works from any Response, regardless of origin (conversation, standalone model.prompt, or from_dict-rehydrated). response.to_dict() / Response.from_dict(data, *, model=None) JSON-safe serialization. to_dict captures model_id, input chain (prompt.messages — full), assembled output (response.messages including reasoning parts and provider_metadata signatures), options, and optional audit fields (id, usage, datetime_utc). from_dict rehydrates a _done=True Response where text() returns the answer and messages returns the full structured view without re-running the assembler. Async variants likewise. Full-chain invariant: response.prompt.messages == what was sent. - Conversation.prompt / AsyncConversation.prompt now pre-compute the full chain (last response's prompt.messages + last response's messages + new user turn, or tool_results for chain loops) and pass it as messages= into the Prompt. - Prompt.messages, when _explicit_messages is set, returns that list verbatim. The previous "combine with prompt= trailing user" behavior is dropped (was Phase 3 scaffolding, no user ever wants a partial chain). - _BaseConversation._build_full_chain is the shared builder. OpenAI adapter simplified: build_messages now reads only prompt.messages and ignores conversation.responses. Under the invariant, prompt.messages already has the full history baked in; walking conversation would double-emit. response.messages now short-circuits to _loaded_messages when set (rehydrated responses don't re-run the assembler). The persistence pattern becomes: response = model.prompt("Hi", thinking=True) response.text() Path("chat.json").write_text(json.dumps(response.to_dict())) # Later, any process: data = json.loads(Path("chat.json").read_text()) response = llm.Response.from_dict(data) response = response.reply("Follow up") print(response.text()) Reasoning signatures (Anthropic extended thinking, Gemini thoughtSignature) round-trip via provider_metadata on the ReasoningPart / ToolCallPart — so multi-turn extended thinking works across process boundaries for free. 19 new tests + 2 tightened existing tests; 600 total pass. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/default_plugins/openai_models.py | 45 +--- llm/models.py | 336 +++++++++++++++++++++++++-- tests/test_openai_messages.py | 146 ++++++++---- tests/test_parts.py | 333 +++++++++++++++++++++++++- 4 files changed, 757 insertions(+), 103 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 7549cd0ae..9a7013390 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -712,42 +712,15 @@ def _append_llm_message(self, out, message, current_system): return current_system def build_messages(self, prompt, conversation): - messages = [] - current_system = None - if conversation is not None: - for prev_response in conversation.responses: - # Input side for the prior turn — read prompt.messages - # so explicit messages= from prior calls round-trips. - for msg in prev_response.prompt.messages: - current_system = self._append_llm_message( - messages, msg, current_system - ) - # Output side — use the flat accessors. They tolerate - # the fact that some plugins mix text and tool_calls at - # the same part_index, which _build_parts would reject. - prev_text = prev_response.text_or_raise() - tool_calls = prev_response.tool_calls_or_raise() - if prev_text or tool_calls: - entry = { - "role": "assistant", - "content": prev_text if prev_text else None, - } - if tool_calls: - entry["tool_calls"] = [ - { - "type": "function", - "id": tc.tool_call_id, - "function": { - "name": tc.name, - "arguments": json.dumps(tc.arguments), - }, - } - for tc in tool_calls - ] - messages.append(entry) - - # Current turn — consume prompt.messages (auto-synthesized from - # legacy kwargs when messages= wasn't explicitly passed). + """Translate prompt.messages into OpenAI's wire format. + + Under the Phase 7 invariant, ``prompt.messages`` is the full + chain for this turn — Conversation.prompt and response.reply + pre-bake the history into it. The ``conversation`` parameter + is unused and retained only for the plugin API contract. + """ + messages: List[Dict[str, Any]] = [] + current_system: Optional[str] = None for msg in prompt.messages: current_system = self._append_llm_message( messages, msg, current_system diff --git a/llm/models.py b/llm/models.py index 6da77b431..d3883ad77 100644 --- a/llm/models.py +++ b/llm/models.py @@ -404,11 +404,24 @@ def system(self): def messages(self): """Canonical list of Message objects for this prompt. - If messages= was passed explicitly, returns that list — with the - optional prompt= text appended as a trailing user TextPart. - Otherwise synthesizes from system=, prompt=, attachments=, and - tool_results= so plugins can read one uniform representation - regardless of which surface the caller used. + **Invariant:** this property returns exactly what the model + was (or will be) sent for this turn — the full chain including + any prior conversation history. + + - If ``messages=`` was passed explicitly, it is authoritative: + returned verbatim. Other kwargs (``prompt=``, ``system=``, + ``attachments=``, ``tool_results=``) are ignored for the + messages list (they remain available via ``prompt.prompt``, + ``prompt.system``, etc., for adapters that still read them). + - Otherwise the list is synthesized from the legacy kwargs + (system, tool_results, prompt, attachments), producing just + the current turn — prior history is not folded in, because + no conversation context is reachable here. + + Conversation.prompt / AsyncConversation.prompt / reply() all + pre-compute the full chain and pass it as ``messages=``, so + ``response.prompt.messages`` after those paths is the full + chain. """ from .parts import ( AttachmentPart, @@ -418,12 +431,7 @@ def messages(self): ) if self._explicit_messages is not None: - out = list(self._explicit_messages) - if self._prompt: - out.append( - Message(role="user", parts=[TextPart(text=self._prompt)]) - ) - return out + return list(self._explicit_messages) result: List["Message"] = [] @@ -486,6 +494,81 @@ class _BaseConversation: def from_row(cls, row: Any) -> "_BaseConversation": raise NotImplementedError + def _build_full_chain( + self, + prompt: Optional[str], + attachments, + tool_results, + explicit_messages, + ) -> List[Any]: + """Build the full message chain for the next turn. + + Walks this conversation's responses to collect prior history, + then appends the new turn's content (explicit messages first, + or synthesized from prompt/attachments/tool_results). + + Returns the list that should be passed as ``messages=`` to the + Prompt constructor so that ``response.prompt.messages`` equals + exactly what the model sees. + + If ``explicit_messages`` is provided, the caller has opted out + of history walking — the list is used as-is. + """ + from .parts import ( + AttachmentPart, + Message, + TextPart, + ToolResultPart, + ) + + if explicit_messages is not None: + return list(explicit_messages) + + chain: List[Any] = [] + for prev in self.responses: + # prev.prompt.messages already contains prev's full input + # chain under the new invariant, but for the FIRST hop into + # a conversation we defensively de-duplicate by only + # concatenating the last response's full chain (which + # transitively includes everything before it). + pass + if self.responses: + last = self.responses[-1] + chain.extend(last.prompt.messages) + # Append that response's own output (structured messages). + try: + chain.extend(last.messages) + except ValueError: + # AsyncResponse not yet awaited — the caller shouldn't + # be constructing a next turn without awaiting first. + pass + + # Append the new turn's input + if tool_results: + chain.append( + Message( + role="tool", + parts=[ + ToolResultPart( + name=tr.name, + output=tr.output, + tool_call_id=tr.tool_call_id, + ) + for tr in tool_results + ], + ) + ) + + user_parts: List[Any] = [] + if prompt: + user_parts.append(TextPart(text=prompt)) + for att in attachments or []: + user_parts.append(AttachmentPart(attachment=att)) + if user_parts: + chain.append(Message(role="user", parts=user_parts)) + + return chain + @dataclass class Conversation(_BaseConversation): @@ -508,6 +591,14 @@ def prompt( key: Optional[str] = None, **options, ) -> "Response": + # Build the authoritative chain so response.prompt.messages + # equals exactly what the model sees for this turn. + chain = self._build_full_chain( + prompt=prompt, + attachments=attachments, + tool_results=tool_results, + explicit_messages=messages, + ) return Response( Prompt( prompt, @@ -519,7 +610,7 @@ def prompt( tools=tools or self.tools, tool_results=tool_results, system_fragments=system_fragments, - messages=messages, + messages=chain, options=self.model.Options(**options), ), self.model, @@ -647,6 +738,12 @@ def prompt( key: Optional[str] = None, **options, ) -> "AsyncResponse": + chain = self._build_full_chain( + prompt=prompt, + attachments=attachments, + tool_results=tool_results, + explicit_messages=messages, + ) return AsyncResponse( Prompt( prompt, @@ -658,7 +755,7 @@ def prompt( tools=tools, tool_results=tool_results, system_fragments=system_fragments, - messages=messages, + messages=chain, options=self.model.Options(**options), ), self.model, @@ -1250,12 +1347,169 @@ def log_to_db(self, db): ) +def _response_to_dict(response: "_BaseResponse") -> Dict[str, Any]: + """Shared serializer for Response.to_dict / AsyncResponse.to_dict. + + The output is a JSON-safe dict — store it anywhere (file, Redis, + Postgres, HTTP body) and round-trip via Response.from_dict. + """ + options = { + key: value + for key, value in dict(response.prompt.options).items() + if value is not None + } + payload: Dict[str, Any] = { + "model": response.model.model_id, + "prompt": { + "messages": [m.to_dict() for m in response.prompt.messages], + }, + "messages": [m.to_dict() for m in response.messages], + } + if options: + payload["prompt"]["options"] = options + if response.prompt._system: + payload["prompt"]["system"] = response.prompt._system + # Optional audit fields — helpful for debugging, not needed for reply(). + if response.id: + payload["id"] = response.id + if response._done: + if response.input_tokens is not None or response.output_tokens is not None: + payload["usage"] = { + "input": response.input_tokens, + "output": response.output_tokens, + "details": response.token_details, + } + if response._start_utcnow is not None: + payload["datetime_utc"] = response._start_utcnow.isoformat() + return payload + + +def _response_from_dict( + data: Dict[str, Any], + cls, + *, + model=None, + async_: bool = False, +) -> "_BaseResponse": + """Shared deserializer for Response.from_dict / AsyncResponse.from_dict.""" + from .parts import Message + + if model is None: + from llm import get_async_model, get_model + + getter = get_async_model if async_ else get_model + model = getter(data["model"]) + + prompt_data = data.get("prompt", {}) + input_messages = [ + Message.from_dict(m) for m in prompt_data.get("messages", []) + ] + output_messages = [ + Message.from_dict(m) for m in data.get("messages", []) + ] + + options_kwargs = prompt_data.get("options") or {} + system = prompt_data.get("system") + + prompt = Prompt( + None, + model=model, + messages=input_messages, + system=system, + options=model.Options(**options_kwargs), + ) + response = cls(prompt, model=model, stream=False) + # Preserve id for audit continuity. + if "id" in data: + response.id = data["id"] + # Rebuild chunks from the assistant's text parts so response.text() + # works without re-running the assembler. + from .parts import TextPart + + response._chunks = [ + p.text + for m in output_messages + for p in m.parts + if isinstance(p, TextPart) and p.text + ] + # Stash the structured output so response.messages returns the + # full picture (reasoning, tool calls, signatures) without needing + # a StreamEvent replay. + response._loaded_messages = output_messages + response._done = True + # Restore usage if present. + usage = data.get("usage") + if usage: + response.input_tokens = usage.get("input") + response.output_tokens = usage.get("output") + response.token_details = usage.get("details") + return response + + class Response(_BaseResponse): "Sync response from a model." model: "Model" conversation: Optional["Conversation"] = None + def reply( + self, + prompt: Optional[str] = None, + *, + messages: Optional[List[Any]] = None, + **kwargs, + ) -> "Response": + """Continue the conversation from this response. + + Builds the next turn's chain as + ``self.prompt.messages + self.messages + [user(prompt)]`` and + calls ``self.model.prompt(messages=chain, ...)``. No + Conversation object required — the Response carries everything + needed. + + If ``messages=`` is passed, its contents are appended to the + chain instead of (or in addition to) the ``prompt`` string. + """ + from .parts import Message, TextPart + + self._force() + chain: List[Any] = list(self.prompt.messages) + list(self.messages) + if prompt: + chain.append( + Message(role="user", parts=[TextPart(text=prompt)]) + ) + if messages: + chain.extend(messages) + return self.model.prompt(messages=chain, **kwargs) + + def to_dict(self) -> Dict[str, Any]: + """Serialize this response for JSON persistence. + + Captures exactly what is needed to continue the conversation: + model id, the input chain that was sent + (``response.prompt.messages``), the structured assistant output + (``response.messages``), and any explicit options. Pair with + :meth:`Response.from_dict` to rehydrate and + :meth:`Response.reply` to continue. + """ + return _response_to_dict(self) + + @classmethod + def from_dict( + cls, + data: Dict[str, Any], + *, + model: Optional["Model"] = None, + ) -> "Response": + """Rehydrate a Response from a ``to_dict()`` payload. + + The returned Response is in the ``_done`` state with + ``response.text()`` and ``response.messages`` populated. + ``model`` overrides the stored model id (useful for continuing + on a different model). + """ + return _response_from_dict(data, cls, model=model, async_=False) + def on_done(self, callback): "Register a callback to be called when the response is complete." if not self._done: @@ -1488,9 +1742,15 @@ def messages(self) -> List[Any]: Almost always a single assistant Message; multiple messages are possible for providers that emit multi-message responses during server-side tool execution (not in this phase's scope). + + Responses rehydrated via ``Response.from_dict`` short-circuit + and return the stored messages directly. """ from .parts import Message + loaded = getattr(self, "_loaded_messages", None) + if loaded is not None: + return list(loaded) self._force() parts = self._build_parts() if not parts: @@ -1510,6 +1770,49 @@ class AsyncResponse(_BaseResponse): model: "AsyncModel" conversation: Optional["AsyncConversation"] = None + def reply( + self, + prompt: Optional[str] = None, + *, + messages: Optional[List[Any]] = None, + **kwargs, + ) -> "AsyncResponse": + """Async counterpart of Response.reply(). Requires this response + to have been awaited (so self.messages is available). + """ + from .parts import Message, TextPart + + if not self._done: + raise ValueError( + "Response not yet awaited — call `await response` before reply()" + ) + chain: List[Any] = list(self.prompt.messages) + list(self.messages) + if prompt: + chain.append( + Message(role="user", parts=[TextPart(text=prompt)]) + ) + if messages: + chain.extend(messages) + return self.model.prompt(messages=chain, **kwargs) + + def to_dict(self) -> Dict[str, Any]: + """Async counterpart of Response.to_dict(). Requires awaiting.""" + if not self._done: + raise ValueError( + "Response not yet awaited — call `await response` before to_dict()" + ) + return _response_to_dict(self) + + @classmethod + def from_dict( + cls, + data: Dict[str, Any], + *, + model: Optional["AsyncModel"] = None, + ) -> "AsyncResponse": + """Async counterpart of Response.from_dict().""" + return _response_from_dict(data, cls, model=model, async_=True) + @classmethod def from_row(cls, db, row, _async=False): return super().from_row(db, row, _async=True) @@ -1776,10 +2079,15 @@ def messages(self) -> List[Any]: """List of Message objects produced by this response. Raises ValueError if the response has not yet been awaited — - assembly depends on the full event stream. + assembly depends on the full event stream. Responses rehydrated + via ``AsyncResponse.from_dict`` short-circuit and return the + stored messages. """ from .parts import Message + loaded = getattr(self, "_loaded_messages", None) + if loaded is not None: + return list(loaded) if not self._done: raise ValueError( "Response not yet awaited — use 'await response' first" diff --git a/tests/test_openai_messages.py b/tests/test_openai_messages.py index 42fff7e14..ca2fc0270 100644 --- a/tests/test_openai_messages.py +++ b/tests/test_openai_messages.py @@ -283,48 +283,39 @@ def test_attachments(self, chat_model): class TestBuildMessagesSystemDedup: - def test_same_system_not_repeated(self, chat_model): - """If two turns share a system prompt, only the first emits it.""" - # Simulate a conversation with a prior response plus a current - # turn; both have the same system prompt. - from llm import Conversation, Response - - conv = Conversation(model=chat_model) - prev_prompt = Prompt( - "first question", model=chat_model, system="be brief" - ) - prev_response = Response(prev_prompt, chat_model, stream=False) - prev_response._chunks = ["first answer"] - prev_response._done = True - conv.responses = [prev_response] + """Explicit messages with repeated system messages dedupe + consecutive identical systems — OpenAI accepts one.""" - new_prompt = Prompt( - "second question", model=chat_model, system="be brief" + def test_same_system_not_repeated(self, chat_model): + prompt = Prompt( + None, + model=chat_model, + messages=[ + llm.system("be brief"), + llm.user("q1"), + llm.assistant("a1"), + llm.system("be brief"), + llm.user("q2"), + ], ) - - result = chat_model.build_messages(new_prompt, conv) - # System appears once. + result = chat_model.build_messages(prompt, None) system_msgs = [m for m in result if m["role"] == "system"] assert len(system_msgs) == 1 assert system_msgs[0]["content"] == "be brief" def test_system_change_emitted(self, chat_model): - from llm import Conversation, Response - - conv = Conversation(model=chat_model) - prev_prompt = Prompt( - "q1", model=chat_model, system="be brief" - ) - prev_response = Response(prev_prompt, chat_model, stream=False) - prev_response._chunks = ["a1"] - prev_response._done = True - conv.responses = [prev_response] - - new_prompt = Prompt( - "q2", model=chat_model, system="be expansive" + prompt = Prompt( + None, + model=chat_model, + messages=[ + llm.system("be brief"), + llm.user("q1"), + llm.assistant("a1"), + llm.system("be expansive"), + llm.user("q2"), + ], ) - - result = chat_model.build_messages(new_prompt, conv) + result = chat_model.build_messages(prompt, None) system_msgs = [m for m in result if m["role"] == "system"] assert [m["content"] for m in system_msgs] == [ "be brief", @@ -334,23 +325,88 @@ def test_system_change_emitted(self, chat_model): class TestBuildMessagesConversationHistory: def test_prior_turn_text_plus_current_user(self, chat_model): - from llm import Conversation, Response - - conv = Conversation(model=chat_model) - prev_prompt = Prompt("what's 1+1?", model=chat_model) - prev_response = Response(prev_prompt, chat_model, stream=False) - prev_response._chunks = ["2"] - prev_response._done = True - conv.responses = [prev_response] - - new_prompt = Prompt("what about 2+2?", model=chat_model) - result = chat_model.build_messages(new_prompt, conv) + """With the Phase 7 invariant, prompt.messages for a follow-up + turn already contains the full chain — the adapter reads only + from it, not from conversation.responses.""" + new_prompt = Prompt( + None, + model=chat_model, + messages=[ + llm.user("what's 1+1?"), + llm.assistant("2"), + llm.user("what about 2+2?"), + ], + ) + result = chat_model.build_messages(new_prompt, None) assert result == [ {"role": "user", "content": "what's 1+1?"}, {"role": "assistant", "content": "2"}, {"role": "user", "content": "what about 2+2?"}, ] + def test_no_double_emission_from_conversation_prompt_flow( + self, chat_model, httpx_mock + ): + """Phase 7 invariant: prompt.messages for a conversation's + follow-up turn is the full chain. The adapter must not ALSO + walk conversation.responses, or the wire body doubles up.""" + # Two staged responses so conv.prompt twice can complete. + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "model": "gpt-4o-mini", + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + "choices": [ + { + "message": {"role": "assistant", "content": "A1"}, + "finish_reason": "stop", + } + ], + }, + headers={"Content-Type": "application/json"}, + ) + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "model": "gpt-4o-mini", + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + "choices": [ + { + "message": {"role": "assistant", "content": "A2"}, + "finish_reason": "stop", + } + ], + }, + headers={"Content-Type": "application/json"}, + ) + + model = llm.get_model("gpt-4o-mini") + conv = model.conversation() + r1 = conv.prompt("Q1", key=API_KEY, stream=False) + r1.text() + r2 = conv.prompt("Q2", key=API_KEY, stream=False) + r2.text() + + # Inspect what was sent on the SECOND turn. + sent_body = json.loads(httpx_mock.get_requests()[-1].content) + sent_messages = sent_body["messages"] + # Exactly three: user(Q1), assistant(A1), user(Q2). + assert sent_messages == [ + {"role": "user", "content": "Q1"}, + {"role": "assistant", "content": "A1"}, + {"role": "user", "content": "Q2"}, + ] + # -- Phase 4b: execute() yields StreamEvents --------------------------- diff --git a/tests/test_parts.py b/tests/test_parts.py index 17cab2027..fdce18809 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -694,18 +694,17 @@ def test_explicit_messages_returned_verbatim(self, mock_model): p = Prompt(None, model=mock_model, messages=explicit) assert p.messages == explicit - def test_explicit_messages_plus_prompt_appends_trailing_user( + def test_explicit_messages_ignores_prompt_kwarg( self, mock_model ): + """Explicit messages= is authoritative. A prompt= string passed + alongside is no longer auto-appended — the invariant is that + prompt.messages equals exactly what the model was sent.""" from llm.models import Prompt - explicit = [llm.system("x"), llm.user("prior")] - p = Prompt("follow-up", model=mock_model, messages=explicit) - assert p.messages == [ - llm.system("x"), - llm.user("prior"), - llm.user("follow-up"), - ] + explicit = [llm.system("x"), llm.user("prior"), llm.user("follow-up")] + p = Prompt("ignored text", model=mock_model, messages=explicit) + assert p.messages == explicit def test_explicit_messages_independent_copy(self, mock_model): """Mutating the caller's list must not mutate Prompt.messages.""" @@ -763,6 +762,324 @@ async def test_async_conversation_prompt_accepts_messages( assert response.prompt.messages == [llm.user("q")] +# -- Phase 7.1: Conversation passes full chain via messages= ---------- +# +# Invariant: response.prompt.messages == exactly what the model was +# sent for this turn, regardless of whether the caller used +# model.prompt(messages=[...]), conversation.prompt("text"), or +# response.reply("text"). + + +class TestConversationFullChainInvariant: + def test_explicit_messages_is_authoritative_no_prompt_combine(self, mock_model): + """Explicit messages= is the whole list. If prompt= is ALSO + passed, it's ignored for messages-building — the caller asked + for exact control.""" + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "this prompt argument is ignored", + messages=[llm.user("q")], + ) + response.text() + assert response.prompt.messages == [llm.user("q")] + + def test_conversation_second_turn_prompt_messages_has_full_chain( + self, mock_model + ): + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + conv = mock_model.conversation() + + r1 = conv.prompt("q1") + r1.text() + r2 = conv.prompt("q2") + r2.text() + + # r2 was sent the full chain. + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + ] + + def test_conversation_third_turn_includes_everything_before( + self, mock_model + ): + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + mock_model.enqueue(["a3"]) + conv = mock_model.conversation() + r1 = conv.prompt("q1"); r1.text() + r2 = conv.prompt("q2"); r2.text() + r3 = conv.prompt("q3"); r3.text() + + assert r3.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + llm.assistant("a2"), + llm.user("q3"), + ] + + def test_conversation_first_turn_chain_is_single_user_message( + self, mock_model + ): + mock_model.enqueue(["a1"]) + conv = mock_model.conversation() + r1 = conv.prompt("q1") + r1.text() + assert r1.prompt.messages == [llm.user("q1")] + + def test_conversation_preserves_reasoning_and_tool_call_parts( + self, mock_model + ): + """The chain carries reasoning and tool calls from prior turns, + not just the flat text — required for multi-turn extended + thinking (Claude) and tool-use round-trips.""" + mock_model.enqueue([ + llm.StreamEvent(type="reasoning", chunk="thinking...", part_index=0), + llm.StreamEvent(type="text", chunk="answer", part_index=1), + ]) + mock_model.enqueue(["follow-up answer"]) + conv = mock_model.conversation() + r1 = conv.prompt("q1") + r1.text() + r2 = conv.prompt("q2") + r2.text() + + assert r2.prompt.messages == [ + llm.user("q1"), + llm.Message( + role="assistant", + parts=[ + llm.ReasoningPart(text="thinking..."), + llm.TextPart(text="answer"), + ], + ), + llm.user("q2"), + ] + + @pytest.mark.asyncio + async def test_async_conversation_full_chain(self, async_mock_model): + async_mock_model.enqueue(["a1"]) + async_mock_model.enqueue(["a2"]) + conv = async_mock_model.conversation() + r1 = conv.prompt("q1") + await r1.text() + r2 = conv.prompt("q2") + await r2.text() + + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + ] + + +# -- Phase 7.3: response.reply() -------------------------------------- + + +class TestResponseReply: + def test_reply_builds_next_turn_from_this_response(self, mock_model): + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + r1 = mock_model.prompt("q1") + r1.text() + + r2 = r1.reply("q2") + r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + ] + + def test_reply_chains(self, mock_model): + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + mock_model.enqueue(["a3"]) + r1 = mock_model.prompt("q1"); r1.text() + r2 = r1.reply("q2"); r2.text() + r3 = r2.reply("q3"); r3.text() + assert r3.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + llm.assistant("a2"), + llm.user("q3"), + ] + + def test_reply_no_prompt_reuses_messages_kwarg(self, mock_model): + """Passing messages= to reply() appends those onto the chain + in place of a new user string.""" + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + r1 = mock_model.prompt("q1") + r1.text() + r2 = r1.reply(messages=[llm.user("alt")]) + r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("alt"), + ] + + def test_reply_from_conversation_response_extends_chain(self, mock_model): + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + conv = mock_model.conversation() + r1 = conv.prompt("q1") + r1.text() + r2 = r1.reply("q2") + r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + ] + + @pytest.mark.asyncio + async def test_async_reply(self, async_mock_model): + async_mock_model.enqueue(["a1"]) + async_mock_model.enqueue(["a2"]) + r1 = async_mock_model.prompt("q1") + await r1.text() + r2 = r1.reply("q2") + await r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + ] + + +# -- Phase 7.2: Response.to_dict / Response.from_dict ------------------ + + +class TestResponseToDictFromDict: + def test_to_dict_captures_chain_and_output(self, mock_model): + mock_model.enqueue(["hello"]) + r = mock_model.prompt("hi") + r.text() + + d = r.to_dict() + assert d["model"] == "mock" + assert d["prompt"]["messages"] == [llm.user("hi").to_dict()] + assert d["messages"] == [llm.assistant("hello").to_dict()] + + def test_from_dict_rehydrates_with_messages(self, mock_model): + mock_model.enqueue(["hello"]) + r = mock_model.prompt("hi") + r.text() + payload = json.dumps(r.to_dict()) + + restored = llm.Response.from_dict(json.loads(payload)) + assert restored._done + assert restored.text() == "hello" + assert restored.messages == [llm.assistant("hello")] + assert restored.prompt.messages == [llm.user("hi")] + + def test_from_dict_then_reply_continues_conversation(self, mock_model): + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + r1 = mock_model.prompt("q1") + r1.text() + + # Serialize across the process boundary + payload = json.dumps(r1.to_dict()) + restored = llm.Response.from_dict(json.loads(payload)) + + # Continue from the restored response + r2 = restored.reply("q2") + r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + ] + + def test_to_dict_preserves_reasoning_and_signatures(self, mock_model): + mock_model.enqueue([ + llm.StreamEvent( + type="reasoning", + chunk="thinking...", + part_index=0, + provider_metadata={"anthropic": {"signature": "sig-abc"}}, + ), + llm.StreamEvent(type="text", chunk="answer", part_index=1), + ]) + r = mock_model.prompt("q") + r.text() + + payload = json.dumps(r.to_dict()) + restored = llm.Response.from_dict(json.loads(payload)) + + msgs = restored.messages + assert msgs[0].role == "assistant" + assert isinstance(msgs[0].parts[0], llm.ReasoningPart) + assert msgs[0].parts[0].text == "thinking..." + assert msgs[0].parts[0].provider_metadata == { + "anthropic": {"signature": "sig-abc"} + } + + def test_from_dict_reply_includes_prior_reasoning_in_chain( + self, mock_model + ): + """The thing this entire refactor was about: a reply() after + from_dict() sends the thinking signature back to the model + for multi-turn extended thinking.""" + mock_model.enqueue([ + llm.StreamEvent( + type="reasoning", + chunk="thinking...", + part_index=0, + provider_metadata={"anthropic": {"signature": "sig-xyz"}}, + ), + llm.StreamEvent(type="text", chunk="answer", part_index=1), + ]) + mock_model.enqueue(["a2"]) + r1 = mock_model.prompt("q1") + r1.text() + + payload = json.dumps(r1.to_dict()) + restored = llm.Response.from_dict(json.loads(payload)) + r2 = restored.reply("q2") + r2.text() + + # The signature must be in the chain sent to the model. + chain = r2.prompt.messages + reasoning_parts = [ + p for m in chain for p in m.parts + if isinstance(p, llm.ReasoningPart) + ] + assert len(reasoning_parts) == 1 + assert reasoning_parts[0].provider_metadata == { + "anthropic": {"signature": "sig-xyz"} + } + + def test_to_dict_captures_options(self, mock_model): + mock_model.enqueue(["ok"]) + r = mock_model.prompt("hi", max_tokens=42) + r.text() + + d = r.to_dict() + assert d["prompt"]["options"] == {"max_tokens": 42} + + def test_from_dict_options_restored(self, mock_model): + mock_model.enqueue(["ok"]) + r = mock_model.prompt("hi", max_tokens=42) + r.text() + + payload = json.dumps(r.to_dict()) + restored = llm.Response.from_dict(json.loads(payload)) + assert restored.prompt.options.max_tokens == 42 + + def test_message_from_dict_static_method_unchanged(self): + # Sanity: Message.from_dict / to_dict keep the Phase 1 contract. + m = llm.assistant("hi") + assert llm.Message.from_dict(m.to_dict()) == m + + class TestChainResponseStreamEvents: def test_sync_chain_stream_events_yields_text_when_no_tools( self, mock_model From 206197d395e2e86ebf547eca330a8bc04af242f4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Apr 2026 08:22:09 -0700 Subject: [PATCH 025/258] Phase 7 follow-up: ChainResponse pre-bakes tool-result chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _chain_for_tool_results() builds the full chain for a tool-result turn inside a chain loop: prior response's full input + output + a tool-role message carrying the new results + any attachments. ChainResponse.responses() and AsyncChainResponse.responses() now pass that chain as messages= when constructing the next Response. Why: under the Phase 7 invariant, response.prompt.messages is what the model sees. Without this, the tool-result turn's prompt.messages would only synthesize a single tool-role message from the legacy tool_results= kwarg — stripping reasoning signatures and tool-call thoughtSignatures from the prior assistant turn. That breaks multi-turn Gemini 3 tool loops (thoughtSignature must be echoed) and Claude extended thinking inside chains. All 600 tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/models.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/llm/models.py b/llm/models.py index d3883ad77..74266b4d7 100644 --- a/llm/models.py +++ b/llm/models.py @@ -2221,6 +2221,57 @@ def __repr__(self): return "".format(self.prompt.prompt, text) +def _chain_for_tool_results( + prior_response, tool_results, attachments +) -> List[Any]: + """Build the message chain for a tool-result turn in a chain loop. + + Takes the prior response's full input chain + its structured + output, then appends a tool-role message carrying the new + ToolResult outputs. Attachments (e.g. images returned by tools) + are folded into a subsequent user-role message. + + This is what gives ``response.prompt.messages`` on the tool- + result turn the complete history for the next provider call — + including any reasoning signatures or thoughtSignatures from the + prior turn. + """ + from .parts import ( + AttachmentPart, + Message, + TextPart, + ToolResultPart, + ) + + chain: List[Any] = list(prior_response.prompt.messages) + list( + prior_response.messages + ) + if tool_results: + chain.append( + Message( + role="tool", + parts=[ + ToolResultPart( + name=tr.name, + output=tr.output, + tool_call_id=tr.tool_call_id, + ) + for tr in tool_results + ], + ) + ) + # Attachments that came back from tools ride on a trailing user + # message (mimics the legacy attachments=[] kwarg behavior). + if attachments: + chain.append( + Message( + role="user", + parts=[AttachmentPart(attachment=a) for a in attachments], + ) + ) + return chain + + class _BaseChainResponse: prompt: "Prompt" stream: bool @@ -2289,12 +2340,20 @@ def responses(self) -> Iterator[Response]: for tool_result in tool_results: attachments.extend(tool_result.attachments) if tool_results: + # Pre-bake the full chain for the tool-result turn so + # response.prompt.messages is what gets sent — carries + # thoughtSignatures, thinking signatures, and everything + # else the model needs for the next call. + next_chain = _chain_for_tool_results( + current_response, tool_results, attachments + ) current_response = Response( Prompt( - "", # Next prompt is empty, tools drive it + "", # Next prompt text is empty; tool_results drive it self.model, tools=current_response.prompt.tools, tool_results=tool_results, + messages=next_chain, options=self.prompt.options, attachments=attachments, ), @@ -2351,11 +2410,17 @@ async def responses(self) -> AsyncIterator[AsyncResponse]: attachments = [] for tool_result in tool_results: attachments.extend(tool_result.attachments) + # Pre-bake chain so prompt.messages carries full history + # + any thinking/tool-call signatures from prior turn. + next_chain = _chain_for_tool_results( + current_response, tool_results, attachments + ) prompt = Prompt( "", self.model, tools=current_response.prompt.tools, tool_results=tool_results, + messages=next_chain, options=self.prompt.options, attachments=attachments, ) From c975d4ce7fd5e1b673a873e8e4b5b99e01e80c4d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Apr 2026 08:37:49 -0700 Subject: [PATCH 026/258] Fix llm -c regression: rehydrated response.messages preserves text+tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _build_parts now falls back to synthesizing from self._chunks and self._tool_calls when self._stream_events is empty. That's the shape of a Response rehydrated via from_row (SQLite doesn't persist StreamEvents under Phase 1-7 scope). Without this, Conversation.prompt's full-chain construction on a follow-up turn (llm -c, load_conversation().prompt(...)) produced [user(q1), user(q2)] — dropping the assistant turn entirely — because prev.messages was []. Now prev.messages yields [assistant(text + tool calls)] and the chain is correct: [user(q1), assistant(a1), user(q2)]. Reasoning signatures and structured reasoning parts are still lost on SQLite rehydrate — that requires Phase 8 (structured parts persistence) or use of response.to_dict() / from_dict() for structure-preserving serialization. For the common case (text-only multi-turn), llm -c works again. Regression tests pin the fallback + the end-to-end load_conversation follow-up. 602 tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/models.py | 39 +++++++++++++++++++++++++++ tests/test_parts.py | 64 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/llm/models.py b/llm/models.py index 74266b4d7..7e8790618 100644 --- a/llm/models.py +++ b/llm/models.py @@ -894,6 +894,14 @@ def _build_parts(self) -> List[Any]: families (text vs tool_call vs reasoning vs tool_result) at the same index is a plugin bug — raises ValueError instead of silently dropping content. + + Fallback: when no stream events were recorded (response was + rehydrated from SQLite via ``from_row``), synthesize a + TextPart from ``self._chunks`` plus any ``self._tool_calls`` + restored by the row loader. Reasoning signatures are not + recoverable from SQLite in this fallback — use + ``response.to_dict()`` / ``Response.from_dict()`` for + structure-preserving persistence. """ from .parts import ( ReasoningPart, @@ -902,6 +910,37 @@ def _build_parts(self) -> List[Any]: ToolResultPart, ) + if not self._stream_events: + # Rehydrated-from-SQLite path: assemble from _chunks + + # _tool_calls so response.messages isn't empty after + # from_row, and Conversation.prompt-built chains include + # the assistant turn on follow-up calls. + parts: List[Any] = [] + text = "".join(self._chunks) + if text: + parts.append(TextPart(text=text)) + for tc in self._tool_calls: + parts.append( + ToolCallPart( + name=tc.name, + arguments=tc.arguments or {}, + tool_call_id=tc.tool_call_id, + ) + ) + reasoning_token_count = getattr( + self, "_reasoning_token_count", 0 + ) + if reasoning_token_count: + parts.insert( + 0, + ReasoningPart( + text="", + redacted=True, + token_count=reasoning_token_count, + ), + ) + return parts + def family(t: str) -> str: if t in ("tool_call_name", "tool_call_args"): return "tool_call" diff --git a/tests/test_parts.py b/tests/test_parts.py index fdce18809..28c459efd 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -876,6 +876,70 @@ async def test_async_conversation_full_chain(self, async_mock_model): ] +# -- Regression: rehydrated-from-SQLite response.messages survives ---- + + +class TestSqliteRehydrateMessages: + """After Response.from_row, response.messages must still yield the + assistant turn as a TextPart (+ any tool calls). Otherwise + Conversation.prompt builds a broken chain for `llm -c`. + """ + + def test_from_row_response_messages_synthesized_from_chunks( + self, mock_model, tmp_path + ): + import sqlite_utils + from llm.migrations import migrate + + mock_model.enqueue(["answer text"]) + r1 = mock_model.prompt("q1") + r1.text() + + db = sqlite_utils.Database(str(tmp_path / "logs.db")) + migrate(db) + r1.log_to_db(db) + + # Rehydrate the response + row = next(db["responses"].rows) + rehydrated = llm.Response.from_row(db, row) + # _stream_events is empty (SQLite doesn't persist those), but + # _chunks carries the text. response.messages must fall back + # to synthesizing a TextPart. + assert rehydrated._stream_events == [] + assert rehydrated.messages == [ + llm.Message(role="assistant", parts=[llm.TextPart(text="answer text")]) + ] + + def test_llm_dash_c_chain_preserves_prior_assistant_turn( + self, mock_model, tmp_path + ): + """End-to-end: a follow-up turn via load_conversation must send + [user(q1), assistant(a1), user(q2)] — not drop the assistant.""" + import sqlite_utils + from llm.migrations import migrate + from llm.cli import load_conversation + + mock_model.enqueue(["first answer"]) + mock_model.enqueue(["second answer"]) + r1 = mock_model.prompt("q1") + r1.text() + + db_path = tmp_path / "logs.db" + db = sqlite_utils.Database(str(db_path)) + migrate(db) + r1.log_to_db(db) + + conv = load_conversation(None, database=str(db_path)) + r2 = conv.prompt("q2") + r2.text() + + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("first answer"), + llm.user("q2"), + ] + + # -- Phase 7.3: response.reply() -------------------------------------- From 82b844d7dd20d9be8112b94a6742801c1541d22e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Apr 2026 09:06:14 -0700 Subject: [PATCH 027/258] Async parity: pin sync/async equivalence for all Phase 1-7 APIs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New tests/test_async_parity.py (18 tests) exercises every new API on the async path via llm-echo (+ async_mock_model where relevant): - AsyncResponse.to_dict() captures chain, output, datetime_utc - AsyncResponse.to_dict() raises before await (guard parity) - AsyncResponse.from_dict() rehydrates and matches original - AsyncResponse.from_dict() + reply() continues correctly - model= override on AsyncResponse.from_dict - AsyncResponse.from_row fallback (SQLite rehydrate) populates response.messages from _chunks so llm -c --async preserves the assistant turn - load_conversation(async_=True).prompt(...) builds full chain - AsyncConversation.chain tool-result turn pre-bakes chain - astream_events matches stream_events for text-only output - reply chains across 3 async turns - Full three-turn save→restore→reply loop under async - reply(messages=[...]) kwarg appends to async chain - response.messages raises on un-awaited AsyncResponse - usage round-trips through async to_dict/from_dict - sync/async structurally identical output for same prompts Plus one test asserting Echo + EchoAsync are both registered. The tests all passed on first run — Phase 7 async implementation was already correct. These pin the invariants against future regressions. 620 tests pass (602 before + 18 new). Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/test_async_parity.py | 385 +++++++++++++++++++++++++++++++++++++ 1 file changed, 385 insertions(+) create mode 100644 tests/test_async_parity.py diff --git a/tests/test_async_parity.py b/tests/test_async_parity.py new file mode 100644 index 000000000..981fbcdcf --- /dev/null +++ b/tests/test_async_parity.py @@ -0,0 +1,385 @@ +"""Async parity: every sync API added in Phases 1-7 must work the same +way on AsyncResponse and AsyncConversation. + +Uses the llm-echo plugin (sync ``Echo`` + async ``EchoAsync``) so both +paths exercise real registered models with identical behaviour. +""" + +import asyncio +import json + +import llm +import pytest + + +# ---- basic sanity: both variants are registered -------------------- + + +def test_echo_registered_for_both(): + assert isinstance(llm.get_model("echo"), llm.Model) + assert isinstance(llm.get_async_model("echo"), llm.AsyncModel) + + +# ---- AsyncResponse.to_dict / from_dict ----------------------------- + + +@pytest.mark.asyncio +async def test_async_to_dict_captures_chain_and_output(): + model = llm.get_async_model("echo") + r = model.prompt("hello") + await r.text() + + d = r.to_dict() + assert d["model"] == "echo" + assert d["prompt"]["messages"] == [llm.user("hello").to_dict()] + # Echo's output is JSON describing the input; it's the assistant's text. + assert len(d["messages"]) == 1 + assert d["messages"][0]["role"] == "assistant" + + +@pytest.mark.asyncio +async def test_async_to_dict_raises_before_awaited(): + model = llm.get_async_model("echo") + r = model.prompt("hello") + with pytest.raises(ValueError): + r.to_dict() + + +@pytest.mark.asyncio +async def test_async_from_dict_rehydrates(): + model = llm.get_async_model("echo") + r = model.prompt("hello") + await r.text() + + payload = json.dumps(r.to_dict()) + restored = llm.AsyncResponse.from_dict(json.loads(payload)) + + assert restored._done + # text_or_raise should match (same text as original) + assert restored.text_or_raise() == r.text_or_raise() + # messages structure preserved + assert restored.messages == r.messages + # prompt.messages (the chain that was sent) preserved + assert restored.prompt.messages == r.prompt.messages + + +@pytest.mark.asyncio +async def test_async_from_dict_then_reply_continues(): + """The whole point: persist an async response across process + boundary (via JSON), rehydrate, continue with reply().""" + model = llm.get_async_model("echo") + r1 = model.prompt("q1") + await r1.text() + + payload = json.dumps(r1.to_dict()) + restored = llm.AsyncResponse.from_dict(json.loads(payload)) + + r2 = restored.reply("q2") + await r2.text() + + # r2 was sent the full chain including r1's output. + chain_roles = [m.role for m in r2.prompt.messages] + assert chain_roles == ["user", "assistant", "user"] + assert r2.prompt.messages[0].parts[0].text == "q1" + assert r2.prompt.messages[-1].parts[0].text == "q2" + + +# ---- AsyncResponse rehydrated via from_row (SQLite path) ----------- + + +@pytest.mark.asyncio +async def test_async_from_row_response_messages_synthesized(tmp_path): + """SQLite rehydrate for async responses must populate + response.messages from _chunks+_tool_calls so follow-up chains + don't silently drop the assistant turn.""" + import sqlite_utils + from llm.migrations import migrate + + model = llm.get_async_model("echo") + r = model.prompt("hello") + await r.text() + + db = sqlite_utils.Database(str(tmp_path / "logs.db")) + migrate(db) + # to_sync_response is what log_to_db uses for async. + sync_r = await r.to_sync_response() + sync_r.log_to_db(db) + + row = next(db["responses"].rows) + rehydrated = llm.AsyncResponse.from_row(db, row) + + assert rehydrated._stream_events == [] + # response.messages falls back to _chunks — must not be empty. + msgs = rehydrated.messages + assert len(msgs) == 1 + assert msgs[0].role == "assistant" + assert isinstance(msgs[0].parts[0], llm.TextPart) + + +# ---- AsyncConversation follow-up via load_conversation ------------- + + +@pytest.mark.asyncio +async def test_async_load_conversation_follow_up_preserves_chain(tmp_path): + """Async equivalent of the llm -c regression: after log_to_db + + load_conversation, a follow-up turn's prompt.messages is the full + [user, assistant, user] chain — not missing the assistant.""" + import sqlite_utils + from llm.cli import load_conversation + from llm.migrations import migrate + + model = llm.get_async_model("echo") + r1 = model.prompt("q1") + await r1.text() + + db_path = tmp_path / "logs.db" + db = sqlite_utils.Database(str(db_path)) + migrate(db) + (await r1.to_sync_response()).log_to_db(db) + + conv = load_conversation(None, async_=True, database=str(db_path)) + r2 = conv.prompt("q2") + await r2.text() + + chain = r2.prompt.messages + assert [m.role for m in chain] == ["user", "assistant", "user"] + assert chain[0].parts[0].text == "q1" + assert chain[-1].parts[0].text == "q2" + + +# ---- Sync/async semantic parity for reply()+to_dict() -------------- + + +def _capture_sync(model): + r1 = model.prompt("ping") + r1.text() + payload1 = json.dumps(r1.to_dict()) + restored = llm.Response.from_dict(json.loads(payload1)) + r2 = restored.reply("pong") + r2.text() + return r2.prompt.messages + + +async def _capture_async(model): + r1 = model.prompt("ping") + await r1.text() + payload1 = json.dumps(r1.to_dict()) + restored = llm.AsyncResponse.from_dict(json.loads(payload1)) + r2 = restored.reply("pong") + await r2.text() + return r2.prompt.messages + + +@pytest.mark.asyncio +async def test_sync_and_async_produce_identical_chain(): + """Run the full save → restore → reply loop against sync Echo and + async EchoAsync. The chain sent on the second turn must be + structurally identical.""" + sync_chain = _capture_sync(llm.get_model("echo")) + async_chain = await _capture_async(llm.get_async_model("echo")) + + # Echo's assistant output differs between invocations only in + # the "previous" field — but for the first turn both see empty + # previous, so outputs match. + sync_dicts = [m.to_dict() for m in sync_chain] + async_dicts = [m.to_dict() for m in async_chain] + assert sync_dicts == async_dicts + + +# ---- AsyncChainResponse tool-result turn pre-bakes chain ----------- + + +@pytest.mark.asyncio +async def test_async_chain_tool_result_turn_has_full_chain(): + """AsyncChainResponse must pre-bake the full chain on tool-result + turns, same as sync ChainResponse.""" + + async def my_tool(x: int) -> int: + "Double the input." + return x * 2 + + model = llm.get_async_model("echo") + # Drive a one-iteration chain by asking echo to emit a tool call + # (echo's JSON-prompt syntax). + chain = model.chain( + json.dumps({ + "tool_calls": [ + {"name": "my_tool", "arguments": {"x": 5}} + ], + "prompt": "prompt", + }), + tools=[llm.Tool.function(my_tool, name="my_tool")], + ) + + responses = [] + async for response in chain.responses(): + responses.append(response) + + # Two responses: the tool-call turn and the tool-result turn. + assert len(responses) == 2 + second = responses[1] + # Second turn's prompt.messages includes the prior turn (user + + # assistant with tool call) plus a tool-role message with the result. + chain_roles = [m.role for m in second.prompt.messages] + assert "tool" in chain_roles + assert chain_roles[0] == "user" + + +# ---- astream_events() parity with stream_events() ------------------ + + +@pytest.mark.asyncio +async def test_astream_events_matches_stream_events_for_text_only(): + """Echo yields plain str (legacy plugin). Both sync and async + paths should wrap those into StreamEvent(type='text') with the + same shape.""" + sync_model = llm.get_model("echo") + async_model = llm.get_async_model("echo") + + sync_r = sync_model.prompt("hello") + sync_events = list(sync_r.stream_events()) + + async_r = async_model.prompt("hello") + async_events = [] + async for ev in async_r.astream_events(): + async_events.append(ev) + + # Same event types, same payload. + assert [e.type for e in sync_events] == [e.type for e in async_events] + assert all(e.type == "text" for e in sync_events) + assert "".join(e.chunk for e in sync_events) == "".join( + e.chunk for e in async_events + ) + + +# ---- Async reply chaining -------------------------------------------- + + +# ---- Additional edge cases ---------------------------------------- + + +@pytest.mark.asyncio +async def test_async_from_dict_model_override(): + model = llm.get_async_model("echo") + r = model.prompt("hi") + await r.text() + payload = json.dumps(r.to_dict()) + + # Pass model explicitly to override whatever's in the payload. + alt = llm.get_async_model("echo") + restored = llm.AsyncResponse.from_dict( + json.loads(payload), model=alt + ) + assert restored.model is alt + + +def test_sync_from_dict_model_override(): + model = llm.get_model("echo") + r = model.prompt("hi") + r.text() + payload = json.dumps(r.to_dict()) + + alt = llm.get_model("echo") + restored = llm.Response.from_dict(json.loads(payload), model=alt) + assert restored.model is alt + + +@pytest.mark.asyncio +async def test_async_to_dict_preserves_datetime(): + model = llm.get_async_model("echo") + r = model.prompt("hi") + await r.text() + d = r.to_dict() + assert "datetime_utc" in d + assert isinstance(d["datetime_utc"], str) + + +@pytest.mark.asyncio +async def test_async_to_dict_preserves_usage_when_set(async_mock_model): + """When a plugin calls response.set_usage, to_dict captures it. + async_mock_model does set usage; llm-echo's async variant doesn't.""" + async_mock_model.enqueue(["ok"]) + r = async_mock_model.prompt("hi") + await r.text() + d = r.to_dict() + assert "usage" in d + assert d["usage"]["input"] is not None + assert d["usage"]["output"] is not None + + # And it round-trips. + restored = llm.AsyncResponse.from_dict(d, model=async_mock_model) + assert restored.input_tokens == d["usage"]["input"] + assert restored.output_tokens == d["usage"]["output"] + + +@pytest.mark.asyncio +async def test_async_reply_messages_kwarg_appends(): + """AsyncResponse.reply(messages=[...]) appends extra messages onto + the chain in place of a trailing user string (mirrors sync test).""" + model = llm.get_async_model("echo") + r1 = model.prompt("q1") + await r1.text() + r2 = r1.reply(messages=[llm.user("extra")]) + await r2.text() + assert [m.role for m in r2.prompt.messages] == ["user", "assistant", "user"] + assert r2.prompt.messages[-1].parts[0].text == "extra" + + +@pytest.mark.asyncio +async def test_async_messages_requires_await_before_to_dict(): + """Parity: accessing response.messages on an un-awaited + AsyncResponse raises, matching to_dict's guard.""" + model = llm.get_async_model("echo") + r = model.prompt("hi") + with pytest.raises(ValueError): + r.messages + + +@pytest.mark.asyncio +async def test_async_full_chain_to_dict_round_trip_three_turns(): + """Serialize on turn 3 — chain must include q1, a1, q2, a2, q3 on + round-trip.""" + model = llm.get_async_model("echo") + r1 = model.prompt("q1") + await r1.text() + r2 = r1.reply("q2") + await r2.text() + r3 = r2.reply("q3") + await r3.text() + + payload = json.dumps(r3.to_dict()) + restored = llm.AsyncResponse.from_dict(json.loads(payload)) + assert [m.role for m in restored.prompt.messages] == [ + "user", "assistant", "user", "assistant", "user" + ] + texts = [m.parts[0].text for m in restored.prompt.messages if m.parts] + assert texts[0] == "q1" + assert texts[2] == "q2" + assert texts[4] == "q3" + + # And continuing from the restored response extends the chain. + r4 = restored.reply("q4") + await r4.text() + assert [m.role for m in r4.prompt.messages] == [ + "user", "assistant", "user", "assistant", "user", "assistant", "user" + ] + + +@pytest.mark.asyncio +async def test_async_reply_chains_three_turns(): + model = llm.get_async_model("echo") + r1 = model.prompt("q1") + await r1.text() + r2 = r1.reply("q2") + await r2.text() + r3 = r2.reply("q3") + await r3.text() + + chain = r3.prompt.messages + assert [m.role for m in chain] == [ + "user", "assistant", "user", "assistant", "user" + ] + texts = [m.parts[0].text for m in chain if m.parts] + assert texts[0] == "q1" + assert texts[2] == "q2" + assert texts[4] == "q3" From 253fed259b3566f9b2bed40171a35c9bd4622f24 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Apr 2026 10:20:29 -0700 Subject: [PATCH 028/258] =?UTF-8?q?Add=20llm/serialization.py=20=E2=80=94?= =?UTF-8?q?=20TypedDicts=20for=20the=20wire=20form?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a dedicated module describing the exact JSON-safe shape returned by Part.to_dict() / Message.to_dict() / Response.to_dict() and accepted by the matching from_dict methods. Every consumer that reads or writes serialized llm data can now import a specific TypedDict and get proper autocomplete, static type-checking, and schema generation support. Module: llm/serialization.py (deliberately not "schema" — that name is taken by the structured-output feature). TextPartDict, ReasoningPartDict, ToolCallPartDict, ToolResultPartDict, AttachmentPartDict — one per Part subclass, each discriminated by a Literal[""] on the `type` field so pydantic/type-checkers can narrow cleanly. PartDict = Union[...] — the discriminated-union form of all Part dicts. AttachmentDict — the nested attachment payload (base64 content when bytes were supplied). MessageDict — {role, parts: list[PartDict], provider_metadata?} PromptDict, UsageDict, ResponseDict — full Response.to_dict() shape including the input chain, options, messages, and audit fields. TypedDicts use typing_extensions.NotRequired (available for 3.10+ via a transitive pydantic dep) so Python 3.10 consumers work. Type annotations on every .to_dict() / .from_dict() method across parts.py and models.py now reference the specific TypedDict rather than Dict[str, Any]. Consumers writing def save_messages(msgs: list[MessageDict]) -> None: ... get autocomplete on msgs[i]["role"], type-errors on typos, and pydantic TypeAdapter-based validation works out of the box: from pydantic import TypeAdapter from llm.serialization import MessageDict TypeAdapter(MessageDict).validate_python(incoming) # validate TypeAdapter(MessageDict).json_schema() # export Also tidied _response_to_dict to omit usage.details when None so the serialized UsageDict doesn't carry a null field where pydantic would reject it during validation. New test_serialization.py (41 tests): - required/optional key sets on every TypedDict - actual .to_dict() output conforms to its TypedDict via TypeAdapter - PartDict discriminated union accepts all 5 Part variants and rejects unknown types - Literal discriminator values are correct - method annotations point at the right TypedDicts - JSON round-trip of Response.to_dict() validates 661 total tests pass (620 before + 41 new). llm-anthropic (32) and llm-gemini (50) still green against the editable llm. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/models.py | 26 +-- llm/parts.py | 50 +++--- llm/serialization.py | 182 ++++++++++++++++++++ tests/test_serialization.py | 319 ++++++++++++++++++++++++++++++++++++ 4 files changed, 547 insertions(+), 30 deletions(-) create mode 100644 llm/serialization.py create mode 100644 tests/test_serialization.py diff --git a/llm/models.py b/llm/models.py index 7e8790618..1a8286cc2 100644 --- a/llm/models.py +++ b/llm/models.py @@ -26,6 +26,7 @@ Union, get_type_hints, ) +from .serialization import ResponseDict from .utils import ( ensure_fragment, ensure_tool, @@ -1386,7 +1387,7 @@ def log_to_db(self, db): ) -def _response_to_dict(response: "_BaseResponse") -> Dict[str, Any]: +def _response_to_dict(response: "_BaseResponse") -> ResponseDict: """Shared serializer for Response.to_dict / AsyncResponse.to_dict. The output is a JSON-safe dict — store it anywhere (file, Redis, @@ -1413,11 +1414,14 @@ def _response_to_dict(response: "_BaseResponse") -> Dict[str, Any]: payload["id"] = response.id if response._done: if response.input_tokens is not None or response.output_tokens is not None: - payload["usage"] = { - "input": response.input_tokens, - "output": response.output_tokens, - "details": response.token_details, - } + usage: Dict[str, Any] = {} + if response.input_tokens is not None: + usage["input"] = response.input_tokens + if response.output_tokens is not None: + usage["output"] = response.output_tokens + if response.token_details is not None: + usage["details"] = response.token_details + payload["usage"] = usage if response._start_utcnow is not None: payload["datetime_utc"] = response._start_utcnow.isoformat() return payload @@ -1521,7 +1525,7 @@ def reply( chain.extend(messages) return self.model.prompt(messages=chain, **kwargs) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> ResponseDict: """Serialize this response for JSON persistence. Captures exactly what is needed to continue the conversation: @@ -1530,13 +1534,15 @@ def to_dict(self) -> Dict[str, Any]: (``response.messages``), and any explicit options. Pair with :meth:`Response.from_dict` to rehydrate and :meth:`Response.reply` to continue. + + Returns :class:`~llm.serialization.ResponseDict`. """ return _response_to_dict(self) @classmethod def from_dict( cls, - data: Dict[str, Any], + data: ResponseDict, *, model: Optional["Model"] = None, ) -> "Response": @@ -1834,7 +1840,7 @@ def reply( chain.extend(messages) return self.model.prompt(messages=chain, **kwargs) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> ResponseDict: """Async counterpart of Response.to_dict(). Requires awaiting.""" if not self._done: raise ValueError( @@ -1845,7 +1851,7 @@ def to_dict(self) -> Dict[str, Any]: @classmethod def from_dict( cls, - data: Dict[str, Any], + data: ResponseDict, *, model: Optional["AsyncModel"] = None, ) -> "AsyncResponse": diff --git a/llm/parts.py b/llm/parts.py index 7a1a83a80..e627f876b 100644 --- a/llm/parts.py +++ b/llm/parts.py @@ -16,9 +16,19 @@ from typing import Any, Dict, List, Optional from .models import Attachment - - -def _attachment_to_dict(att: Attachment) -> Dict[str, Any]: +from .serialization import ( + AttachmentDict, + AttachmentPartDict, + MessageDict, + PartDict, + ReasoningPartDict, + TextPartDict, + ToolCallPartDict, + ToolResultPartDict, +) + + +def _attachment_to_dict(att: Attachment) -> AttachmentDict: d: Dict[str, Any] = {} if att.type: d["type"] = att.type @@ -28,10 +38,10 @@ def _attachment_to_dict(att: Attachment) -> Dict[str, Any]: d["path"] = att.path if att.content: d["content"] = base64.b64encode(att.content).decode("ascii") - return d + return d # type: ignore[return-value] -def _attachment_from_dict(d: Dict[str, Any]) -> Attachment: +def _attachment_from_dict(d: AttachmentDict) -> Attachment: content = d.get("content") if isinstance(content, str): content = base64.b64decode(content) @@ -47,11 +57,11 @@ def _attachment_from_dict(d: Dict[str, Any]) -> Attachment: class Part: """Base class for all parts. Role lives on the enclosing Message.""" - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> PartDict: raise NotImplementedError @staticmethod - def from_dict(d: Dict[str, Any]) -> "Part": + def from_dict(d: PartDict) -> "Part": type_ = d.get("type") pm = d.get("provider_metadata") if type_ == "text": @@ -95,11 +105,11 @@ class TextPart(Part): text: str = "" provider_metadata: Optional[Dict[str, Any]] = None - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> TextPartDict: d: Dict[str, Any] = {"type": "text", "text": self.text} if self.provider_metadata: d["provider_metadata"] = self.provider_metadata - return d + return d # type: ignore[return-value] @dataclass @@ -116,7 +126,7 @@ class ReasoningPart(Part): token_count: Optional[int] = None provider_metadata: Optional[Dict[str, Any]] = None - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> ReasoningPartDict: d: Dict[str, Any] = {"type": "reasoning", "text": self.text} if self.redacted: d["redacted"] = True @@ -124,7 +134,7 @@ def to_dict(self) -> Dict[str, Any]: d["token_count"] = self.token_count if self.provider_metadata: d["provider_metadata"] = self.provider_metadata - return d + return d # type: ignore[return-value] @dataclass @@ -142,7 +152,7 @@ class ToolCallPart(Part): server_executed: bool = False provider_metadata: Optional[Dict[str, Any]] = None - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> ToolCallPartDict: d: Dict[str, Any] = { "type": "tool_call", "name": self.name, @@ -154,7 +164,7 @@ def to_dict(self) -> Dict[str, Any]: d["server_executed"] = True if self.provider_metadata: d["provider_metadata"] = self.provider_metadata - return d + return d # type: ignore[return-value] @dataclass @@ -169,7 +179,7 @@ class ToolResultPart(Part): exception: Optional[str] = None provider_metadata: Optional[Dict[str, Any]] = None - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> ToolResultPartDict: d: Dict[str, Any] = { "type": "tool_result", "name": self.name, @@ -185,7 +195,7 @@ def to_dict(self) -> Dict[str, Any]: d["attachments"] = [_attachment_to_dict(a) for a in self.attachments] if self.provider_metadata: d["provider_metadata"] = self.provider_metadata - return d + return d # type: ignore[return-value] @dataclass @@ -195,13 +205,13 @@ class AttachmentPart(Part): attachment: Optional[Attachment] = None provider_metadata: Optional[Dict[str, Any]] = None - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> AttachmentPartDict: d: Dict[str, Any] = {"type": "attachment"} if self.attachment: d["attachment"] = _attachment_to_dict(self.attachment) if self.provider_metadata: d["provider_metadata"] = self.provider_metadata - return d + return d # type: ignore[return-value] @dataclass @@ -218,17 +228,17 @@ class Message: parts: List[Part] = field(default_factory=list) provider_metadata: Optional[Dict[str, Any]] = None - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> MessageDict: d: Dict[str, Any] = { "role": self.role, "parts": [p.to_dict() for p in self.parts], } if self.provider_metadata: d["provider_metadata"] = self.provider_metadata - return d + return d # type: ignore[return-value] @staticmethod - def from_dict(d: Dict[str, Any]) -> "Message": + def from_dict(d: MessageDict) -> "Message": return Message( role=d["role"], parts=[Part.from_dict(p) for p in d.get("parts", [])], diff --git a/llm/serialization.py b/llm/serialization.py new file mode 100644 index 000000000..b6b4bf972 --- /dev/null +++ b/llm/serialization.py @@ -0,0 +1,182 @@ +"""TypedDict spec for the JSON-safe wire form of Part, Message, and Response. + +These are the exact shapes returned by ``Part.to_dict()``, +``Message.to_dict()``, and ``Response.to_dict()`` — and accepted by the +matching ``from_dict`` classmethods. They are the canonical wire format; +use them to annotate any code that reads or writes serialized llm data. + +Example:: + + from llm.serialization import MessageDict + + def save_messages(conn, messages: list[MessageDict]) -> None: + for m in messages: + conn.execute( + "INSERT INTO messages(role, parts_json) VALUES (?, ?)", + (m["role"], json.dumps(m["parts"])), + ) + +Or pair with Pydantic's TypeAdapter for runtime validation:: + + from pydantic import TypeAdapter + from llm.serialization import MessageDict + + msg = TypeAdapter(MessageDict).validate_python(incoming_dict) + +Or export JSON Schema for cross-language consumers:: + + schema = TypeAdapter(MessageDict).json_schema() + +The TypedDicts are erased at runtime — zero overhead. ``NotRequired`` +keys may be absent from a serialized payload; required keys must +always be present. +""" + +from typing import Any, Dict, List, Literal, Union + +# NotRequired moved to typing in 3.11; use typing_extensions for 3.10 +# support. typing_extensions is a transitive dep via pydantic. +from typing_extensions import NotRequired, TypedDict + + +__all__ = [ + "AttachmentDict", + "AttachmentPartDict", + "MessageDict", + "PartDict", + "PromptDict", + "ReasoningPartDict", + "ResponseDict", + "TextPartDict", + "ToolCallPartDict", + "ToolResultPartDict", + "UsageDict", +] + + +# ---- Attachment payload (nested inside AttachmentPartDict + tool results) ---- + + +class AttachmentDict(TypedDict, total=False): + """Nested attachment payload. All fields optional — an Attachment + may carry a type, a url, a path, and/or base64-encoded content. + """ + + type: str + url: str + path: str + # base64-encoded bytes when the attachment was constructed with raw + # content= bytes. + content: str + + +# ---- Per-Part TypedDicts (discriminated by the `type` field) ----------------- + + +class TextPartDict(TypedDict): + type: Literal["text"] + text: str + provider_metadata: NotRequired[Dict[str, Any]] + + +class ReasoningPartDict(TypedDict): + type: Literal["reasoning"] + text: str + # Redacted reasoning: text is "" and token_count carries the opaque + # count reported by the provider (OpenAI GPT-5, Gemini thinking). + redacted: NotRequired[bool] + token_count: NotRequired[int] + provider_metadata: NotRequired[Dict[str, Any]] + + +class ToolCallPartDict(TypedDict): + type: Literal["tool_call"] + name: str + arguments: Dict[str, Any] + tool_call_id: NotRequired[str] + # True for provider-executed calls (Anthropic web search, Gemini code + # execution). Client echoes the block back as-is on next turn. + server_executed: NotRequired[bool] + provider_metadata: NotRequired[Dict[str, Any]] + + +class ToolResultPartDict(TypedDict): + type: Literal["tool_result"] + name: str + output: str + tool_call_id: NotRequired[str] + server_executed: NotRequired[bool] + exception: NotRequired[str] + attachments: NotRequired[List[AttachmentDict]] + provider_metadata: NotRequired[Dict[str, Any]] + + +class AttachmentPartDict(TypedDict): + type: Literal["attachment"] + attachment: NotRequired[AttachmentDict] + provider_metadata: NotRequired[Dict[str, Any]] + + +PartDict = Union[ + TextPartDict, + ReasoningPartDict, + ToolCallPartDict, + ToolResultPartDict, + AttachmentPartDict, +] +"""Discriminated union of Part dict shapes. Use with +``pydantic.TypeAdapter(PartDict)`` to validate / dispatch by ``type``. +""" + + +# ---- Message ---------------------------------------------------------------- + + +class MessageDict(TypedDict): + """JSON-safe form of ``llm.Message``. + + ``role`` is one of "user", "assistant", "system", "tool" in practice + — typed as ``str`` here to leave room for provider-specific values. + """ + + role: str + parts: List[PartDict] + provider_metadata: NotRequired[Dict[str, Any]] + + +# ---- Response + nested shapes ----------------------------------------------- + + +class PromptDict(TypedDict): + """The ``prompt`` sub-dict of ``Response.to_dict()`` — captures the + full input chain that was sent for this turn plus any options that + apply.""" + + messages: List[MessageDict] + options: NotRequired[Dict[str, Any]] + system: NotRequired[str] + + +class UsageDict(TypedDict, total=False): + """Optional usage block on ``ResponseDict``. All fields optional; + providers vary in which they report.""" + + input: int + output: int + details: Dict[str, Any] + + +class ResponseDict(TypedDict): + """JSON-safe form of ``llm.Response`` — everything needed for + ``Response.from_dict`` to rehydrate and ``response.reply()`` to + continue a conversation across a process boundary. + """ + + model: str + prompt: PromptDict + messages: List[MessageDict] + # Audit fields — present on a freshly-serialized response, optional + # on hand-constructed ones. + id: NotRequired[str] + usage: NotRequired[UsageDict] + datetime_utc: NotRequired[str] diff --git a/tests/test_serialization.py b/tests/test_serialization.py new file mode 100644 index 000000000..9f3313bfd --- /dev/null +++ b/tests/test_serialization.py @@ -0,0 +1,319 @@ +"""Tests for llm.serialization — the TypedDict spec for the JSON-safe +wire form of Message, Part, and Response. + +Uses pydantic.TypeAdapter to verify that actual to_dict() output +conforms to the TypedDict annotations. pydantic is already a runtime +dependency. +""" + +import json +import pytest +from pydantic import TypeAdapter + +import llm +from llm.serialization import ( + AttachmentDict, + AttachmentPartDict, + MessageDict, + PartDict, + PromptDict, + ResponseDict, + ReasoningPartDict, + TextPartDict, + ToolCallPartDict, + ToolResultPartDict, + UsageDict, +) + + +# ---- required/optional keys ---------------------------------------- + + +class TestRequiredOptionalKeys: + def test_message_dict_required_keys(self): + assert MessageDict.__required_keys__ == {"role", "parts"} + assert MessageDict.__optional_keys__ == {"provider_metadata"} + + def test_text_part_dict_required_keys(self): + assert TextPartDict.__required_keys__ == {"type", "text"} + assert TextPartDict.__optional_keys__ == {"provider_metadata"} + + def test_reasoning_part_dict_required_keys(self): + assert ReasoningPartDict.__required_keys__ == {"type", "text"} + assert ReasoningPartDict.__optional_keys__ == { + "redacted", "token_count", "provider_metadata" + } + + def test_tool_call_part_dict_required_keys(self): + assert ToolCallPartDict.__required_keys__ == { + "type", "name", "arguments" + } + assert ToolCallPartDict.__optional_keys__ == { + "tool_call_id", "server_executed", "provider_metadata" + } + + def test_tool_result_part_dict_required_keys(self): + assert ToolResultPartDict.__required_keys__ == { + "type", "name", "output" + } + assert ToolResultPartDict.__optional_keys__ == { + "tool_call_id", "server_executed", "exception", + "attachments", "provider_metadata", + } + + def test_attachment_part_dict_required_keys(self): + assert AttachmentPartDict.__required_keys__ == {"type"} + assert AttachmentPartDict.__optional_keys__ == { + "attachment", "provider_metadata" + } + + def test_response_dict_required_keys(self): + assert ResponseDict.__required_keys__ == {"model", "prompt", "messages"} + assert ResponseDict.__optional_keys__ == {"id", "usage", "datetime_utc"} + + +# ---- to_dict output conforms to the TypedDict ---------------------- + + +class TestPartRoundTrip: + def _adapter(self, td): + return TypeAdapter(td) + + def test_text_part_matches(self): + d = llm.TextPart(text="hello").to_dict() + self._adapter(TextPartDict).validate_python(d) + + def test_text_part_with_provider_metadata_matches(self): + d = llm.TextPart( + text="hi", provider_metadata={"anthropic": {"cached": True}} + ).to_dict() + self._adapter(TextPartDict).validate_python(d) + + def test_reasoning_part_redacted_matches(self): + d = llm.ReasoningPart( + text="", redacted=True, token_count=150 + ).to_dict() + self._adapter(ReasoningPartDict).validate_python(d) + + def test_reasoning_part_with_signature_matches(self): + d = llm.ReasoningPart( + text="thinking...", + provider_metadata={"anthropic": {"signature": "sig-abc"}}, + ).to_dict() + self._adapter(ReasoningPartDict).validate_python(d) + + def test_tool_call_part_matches(self): + d = llm.ToolCallPart( + name="search", arguments={"q": "x"}, tool_call_id="c1" + ).to_dict() + self._adapter(ToolCallPartDict).validate_python(d) + + def test_tool_result_part_matches(self): + d = llm.ToolResultPart( + name="search", output="result", tool_call_id="c1" + ).to_dict() + self._adapter(ToolResultPartDict).validate_python(d) + + def test_attachment_part_with_url_matches(self): + att = llm.Attachment(type="image/jpeg", url="https://example.com/cat.jpg") + d = llm.AttachmentPart(attachment=att).to_dict() + self._adapter(AttachmentPartDict).validate_python(d) + + def test_attachment_part_with_bytes_matches(self): + att = llm.Attachment(type="image/png", content=b"\x89PNG...") + d = llm.AttachmentPart(attachment=att).to_dict() + self._adapter(AttachmentPartDict).validate_python(d) + + +class TestPartDiscriminatedUnion: + def test_text_part_validates_as_part_dict(self): + d = llm.TextPart(text="hi").to_dict() + TypeAdapter(PartDict).validate_python(d) + + def test_reasoning_part_validates_as_part_dict(self): + d = llm.ReasoningPart(text="thinking").to_dict() + TypeAdapter(PartDict).validate_python(d) + + def test_tool_call_part_validates_as_part_dict(self): + d = llm.ToolCallPart( + name="t", arguments={}, tool_call_id="c1" + ).to_dict() + TypeAdapter(PartDict).validate_python(d) + + def test_tool_result_part_validates_as_part_dict(self): + d = llm.ToolResultPart( + name="t", output="out", tool_call_id="c1" + ).to_dict() + TypeAdapter(PartDict).validate_python(d) + + def test_attachment_part_validates_as_part_dict(self): + att = llm.Attachment(type="image/jpeg", url="http://x") + d = llm.AttachmentPart(attachment=att).to_dict() + TypeAdapter(PartDict).validate_python(d) + + def test_unknown_type_rejected(self): + with pytest.raises(Exception): + TypeAdapter(PartDict).validate_python( + {"type": "nonsense", "text": "x"} + ) + + +class TestMessageDictRoundTrip: + def test_user_message_matches(self): + d = llm.user("hi").to_dict() + TypeAdapter(MessageDict).validate_python(d) + + def test_assistant_with_mixed_parts_matches(self): + m = llm.Message( + role="assistant", + parts=[ + llm.ReasoningPart( + text="thinking", + provider_metadata={"anthropic": {"signature": "s"}}, + ), + llm.TextPart(text="answer"), + llm.ToolCallPart( + name="search", + arguments={"q": "x"}, + tool_call_id="c1", + ), + ], + ) + TypeAdapter(MessageDict).validate_python(m.to_dict()) + + def test_tool_role_message_with_results_matches(self): + m = llm.tool_message( + llm.ToolResultPart(name="s", output="r", tool_call_id="c1"), + ) + TypeAdapter(MessageDict).validate_python(m.to_dict()) + + +class TestResponseDictRoundTrip: + def test_mock_response_to_dict_matches(self, mock_model): + mock_model.enqueue(["answer"]) + r = mock_model.prompt("q") + r.text() + + d = r.to_dict() + TypeAdapter(ResponseDict).validate_python(d) + + def test_response_with_reasoning_matches(self, mock_model): + mock_model.enqueue([ + llm.StreamEvent( + type="reasoning", + chunk="thinking", + part_index=0, + provider_metadata={"anthropic": {"signature": "s"}}, + ), + llm.StreamEvent(type="text", chunk="answer", part_index=1), + ]) + r = mock_model.prompt("q") + r.text() + + d = r.to_dict() + TypeAdapter(ResponseDict).validate_python(d) + + def test_response_with_options_matches(self, mock_model): + mock_model.enqueue(["ok"]) + r = mock_model.prompt("q", max_tokens=42) + r.text() + + d = r.to_dict() + TypeAdapter(ResponseDict).validate_python(d) + assert d["prompt"].get("options") == {"max_tokens": 42} + + +# ---- Literal discriminators ---------------------------------------- + + +class TestLiteralDiscriminators: + """The `type` field on each PartDict is a Literal — that's how + Pydantic's discriminated unions work. Verify each literal.""" + + def test_text_part_literal_is_text(self): + import typing + hints = typing.get_type_hints(TextPartDict) + # Literal["text"] — check the args + assert typing.get_args(hints["type"]) == ("text",) + + def test_reasoning_part_literal_is_reasoning(self): + import typing + hints = typing.get_type_hints(ReasoningPartDict) + assert typing.get_args(hints["type"]) == ("reasoning",) + + def test_tool_call_part_literal_is_tool_call(self): + import typing + hints = typing.get_type_hints(ToolCallPartDict) + assert typing.get_args(hints["type"]) == ("tool_call",) + + def test_tool_result_part_literal_is_tool_result(self): + import typing + hints = typing.get_type_hints(ToolResultPartDict) + assert typing.get_args(hints["type"]) == ("tool_result",) + + def test_attachment_part_literal_is_attachment(self): + import typing + hints = typing.get_type_hints(AttachmentPartDict) + assert typing.get_args(hints["type"]) == ("attachment",) + + +# ---- to_dict / from_dict return-type annotations ------------------- + + +class TestAnnotations: + """Method signatures should advertise the specific TypedDicts.""" + + def test_text_part_to_dict_annotation(self): + import typing + hints = typing.get_type_hints(llm.TextPart.to_dict) + assert hints["return"] is TextPartDict + + def test_reasoning_part_to_dict_annotation(self): + import typing + hints = typing.get_type_hints(llm.ReasoningPart.to_dict) + assert hints["return"] is ReasoningPartDict + + def test_tool_call_part_to_dict_annotation(self): + import typing + hints = typing.get_type_hints(llm.ToolCallPart.to_dict) + assert hints["return"] is ToolCallPartDict + + def test_tool_result_part_to_dict_annotation(self): + import typing + hints = typing.get_type_hints(llm.ToolResultPart.to_dict) + assert hints["return"] is ToolResultPartDict + + def test_attachment_part_to_dict_annotation(self): + import typing + hints = typing.get_type_hints(llm.AttachmentPart.to_dict) + assert hints["return"] is AttachmentPartDict + + def test_message_to_dict_annotation(self): + import typing + hints = typing.get_type_hints(llm.Message.to_dict) + assert hints["return"] is MessageDict + + def test_message_from_dict_annotation(self): + import typing + hints = typing.get_type_hints(llm.Message.from_dict) + assert hints["d"] is MessageDict + + def test_response_to_dict_annotation(self): + import typing + hints = typing.get_type_hints(llm.Response.to_dict) + assert hints["return"] is ResponseDict + + +# ---- End-to-end JSON round-trip validates against schema ----------- + + +class TestEndToEnd: + def test_json_roundtrip_validates(self, mock_model): + mock_model.enqueue(["text answer"]) + r = mock_model.prompt("q") + r.text() + + payload = json.dumps(r.to_dict()) + parsed = json.loads(payload) + # Parsed dict should still conform to ResponseDict. + TypeAdapter(ResponseDict).validate_python(parsed) From bb5daaf6d86efd01bd35e5121ecbfe433168ad40 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Apr 2026 15:15:39 -0700 Subject: [PATCH 029/258] Add messages= parameter to chain() (sync + async, Conversation + Model) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parity with prompt(): all four chain() methods — Conversation.chain, AsyncConversation.chain, _Model.chain, _AsyncModel.chain — now accept a messages= kwarg and pre-bake the full chain via _build_full_chain so the first response of the chain loop satisfies the invariant response.prompt.messages == what was sent. Semantics match prompt() exactly: when messages= is passed, it's authoritative for the first turn. The prompt= kwarg is ignored for chain construction (it stays available via prompt.prompt / .system / .attachments for any legacy plugin code). Subsequent tool-result turns inside the chain loop still extend the chain via _chain_for_tool_results, which reads from the prior response's prompt.messages + messages. Six new tests cover: conv.chain(messages=), model.chain(messages=), messages= authoritative over prompt= kwarg, explicit messages= replaces conversation history, and the async variants of all of the above. 667 tests in llm core (661 before + 6 new); llm-anthropic (32) and llm-gemini (50) still green. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/models.py | 24 +++++++++++++++ tests/test_parts.py | 73 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/llm/models.py b/llm/models.py index 1a8286cc2..ed5ccab36 100644 --- a/llm/models.py +++ b/llm/models.py @@ -628,6 +628,7 @@ def chain( attachments: Optional[List[Attachment]] = None, system: Optional[str] = None, system_fragments: Optional[List[str]] = None, + messages: Optional[List[Any]] = None, stream: bool = True, schema: Optional[Union[dict, type[BaseModel]]] = None, tools: Optional[List[ToolDef]] = None, @@ -639,6 +640,16 @@ def chain( options: Optional[dict] = None, ) -> "ChainResponse": self.model._validate_attachments(attachments) + # Parity with Conversation.prompt: pre-bake the full chain so + # response.prompt.messages is authoritative for the first turn + # of the chain loop. Subsequent tool-result turns extend the + # chain via _chain_for_tool_results. + chain_messages = self._build_full_chain( + prompt=prompt, + attachments=attachments, + tool_results=tool_results, + explicit_messages=messages, + ) return ChainResponse( Prompt( prompt, @@ -649,6 +660,7 @@ def chain( tools=tools or self.tools, tool_results=tool_results, system_fragments=system_fragments, + messages=chain_messages, model=self.model, options=self.model.Options(**(options or {})), ), @@ -690,6 +702,7 @@ def chain( attachments: Optional[List[Attachment]] = None, system: Optional[str] = None, system_fragments: Optional[List[str]] = None, + messages: Optional[List[Any]] = None, stream: bool = True, schema: Optional[Union[dict, type[BaseModel]]] = None, tools: Optional[List[ToolDef]] = None, @@ -701,6 +714,12 @@ def chain( options: Optional[dict] = None, ) -> "AsyncChainResponse": self.model._validate_attachments(attachments) + chain_messages = self._build_full_chain( + prompt=prompt, + attachments=attachments, + tool_results=tool_results, + explicit_messages=messages, + ) return AsyncChainResponse( Prompt( prompt, @@ -711,6 +730,7 @@ def chain( tools=tools or self.tools, tool_results=tool_results, system_fragments=system_fragments, + messages=chain_messages, model=self.model, options=self.model.Options(**(options or {})), ), @@ -2634,6 +2654,7 @@ def chain( attachments: Optional[List[Attachment]] = None, system: Optional[str] = None, system_fragments: Optional[List[str]] = None, + messages: Optional[List[Any]] = None, stream: bool = True, schema: Optional[Union[dict, type[BaseModel]]] = None, tools: Optional[List[ToolDef]] = None, @@ -2649,6 +2670,7 @@ def chain( attachments=attachments, system=system, system_fragments=system_fragments, + messages=messages, stream=stream, schema=schema, tools=tools, @@ -2745,6 +2767,7 @@ def chain( attachments: Optional[List[Attachment]] = None, system: Optional[str] = None, system_fragments: Optional[List[str]] = None, + messages: Optional[List[Any]] = None, stream: bool = True, schema: Optional[Union[dict, type[BaseModel]]] = None, tools: Optional[List[ToolDef]] = None, @@ -2760,6 +2783,7 @@ def chain( attachments=attachments, system=system, system_fragments=system_fragments, + messages=messages, stream=stream, schema=schema, tools=tools, diff --git a/tests/test_parts.py b/tests/test_parts.py index 28c459efd..b8db266f4 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -1017,6 +1017,79 @@ async def test_async_reply(self, async_mock_model): ] +# -- chain() accepts messages= (parity with prompt()) ----------------- + + +class TestChainMessagesKwarg: + def test_conversation_chain_accepts_messages(self, mock_model): + mock_model.enqueue(["ok"]) + conv = mock_model.conversation() + chain = conv.chain(messages=[llm.user("explicit")]) + chain.text() + r1 = chain._responses[0] + assert r1.prompt.messages == [llm.user("explicit")] + + def test_model_chain_accepts_messages(self, mock_model): + mock_model.enqueue(["ok"]) + chain = mock_model.chain(messages=[llm.user("explicit")]) + chain.text() + r1 = chain._responses[0] + assert r1.prompt.messages == [llm.user("explicit")] + + def test_chain_messages_is_authoritative_over_prompt_kwarg( + self, mock_model + ): + """Parity with prompt(): when both are passed, messages= wins + and the prompt= string is not folded into the chain.""" + mock_model.enqueue(["ok"]) + chain = mock_model.chain( + "ignored text", + messages=[llm.user("explicit")], + ) + chain.text() + r1 = chain._responses[0] + assert r1.prompt.messages == [llm.user("explicit")] + + def test_chain_with_messages_and_prior_conversation(self, mock_model): + """Explicit messages= on chain() replaces any history walking — + the chain starts from that exact list.""" + mock_model.enqueue(["first"]) + mock_model.enqueue(["second"]) + conv = mock_model.conversation() + r1 = conv.prompt("prior") + r1.text() + + # Now start a chain with explicit messages= — prior turn is + # ignored (consistent with prompt() behavior). + chain = conv.chain(messages=[llm.user("fresh start")]) + chain.text() + first_chain_response = chain._responses[0] + assert first_chain_response.prompt.messages == [ + llm.user("fresh start") + ] + + @pytest.mark.asyncio + async def test_async_conversation_chain_accepts_messages( + self, async_mock_model + ): + async_mock_model.enqueue(["ok"]) + conv = async_mock_model.conversation() + chain = conv.chain(messages=[llm.user("explicit")]) + await chain.text() + r1 = chain._responses[0] + assert r1.prompt.messages == [llm.user("explicit")] + + @pytest.mark.asyncio + async def test_async_model_chain_accepts_messages( + self, async_mock_model + ): + async_mock_model.enqueue(["ok"]) + chain = async_mock_model.chain(messages=[llm.user("explicit")]) + await chain.text() + r1 = chain._responses[0] + assert r1.prompt.messages == [llm.user("explicit")] + + # -- Phase 7.2: Response.to_dict / Response.from_dict ------------------ From 3df8e4426bca8e6d14b9f4c2bb7d2196c1b86af5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Apr 2026 15:55:21 -0700 Subject: [PATCH 030/258] Chain tool-result turns now carry system + system_fragments forward MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: ChainResponse.responses() and AsyncChainResponse.responses() built the follow-up Prompt (for tool-result turns inside the chain loop) without propagating system= or system_fragments= from the initial prompt. Adapters that read prompt.system directly — OpenAI's Chat, for example, which sends system as its own message — saw an empty system on every turn after the first, silently losing the caller's instruction. Fix: pass system=self.prompt._system and system_fragments=self.prompt.system_fragments when constructing the next Prompt. Same change on sync and async paths. _chain_for_tool_results keeps building messages= from the prior response's prompt.messages + messages, so adapters that read prompt.messages (the Phase 7 canonical input) continue to work too — this fix is specifically for the adapters that still use the prompt.system legacy field. Three regression tests (sync, sync+system_fragments, async) pin the new behavior. 670 tests in llm core; llm-anthropic (32) and llm-gemini (50) still green. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/models.py | 11 +++++ tests/test_parts.py | 117 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) diff --git a/llm/models.py b/llm/models.py index ed5ccab36..8600eb402 100644 --- a/llm/models.py +++ b/llm/models.py @@ -2419,6 +2419,13 @@ def responses(self) -> Iterator[Response]: tools=current_response.prompt.tools, tool_results=tool_results, messages=next_chain, + # Carry system + system_fragments forward so + # stateless-per-turn adapters (OpenAI and + # friends that read prompt.system directly) + # keep seeing the system prompt on every call + # of the chain loop. + system=self.prompt._system, + system_fragments=self.prompt.system_fragments, options=self.prompt.options, attachments=attachments, ), @@ -2486,6 +2493,10 @@ async def responses(self) -> AsyncIterator[AsyncResponse]: tools=current_response.prompt.tools, tool_results=tool_results, messages=next_chain, + # Carry system + system_fragments forward — same + # reasoning as the sync path. + system=self.prompt._system, + system_fragments=self.prompt.system_fragments, options=self.prompt.options, attachments=attachments, ) diff --git a/tests/test_parts.py b/tests/test_parts.py index b8db266f4..9cee6b672 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -1017,6 +1017,123 @@ async def test_async_reply(self, async_mock_model): ] +# -- chain() propagates system across tool-result turns -------------- + + +class TestChainPropagatesSystem: + """On a tool-result turn within a chain loop, the Prompt must + carry forward the original system= and system_fragments= so + adapters that read prompt.system (OpenAI and other + stateless-per-turn providers) see it on every call.""" + + def test_sync_chain_tool_result_turn_preserves_system(self, mock_model): + # First turn: fake a tool call so the chain iterates. + tool_call = llm.ToolCall( + tool_call_id="c1", name="tick", arguments={} + ) + + class ChainMock(type(mock_model)): + def execute(self, prompt, stream, response, conversation): + if not self._queue: + yield "done" + return + msgs = self._queue.pop(0) + for m in msgs: + yield m + if not response._tool_calls: + response.add_tool_call(tool_call) + + def tick() -> str: + "Tick" + return "tock" + + m = ChainMock() + m.enqueue(["tool-turn"]) # first response; chain will loop + m.enqueue(["final"]) # second response, after tool results + + chain = m.chain("q", system="be brief", tools=[tick]) + list(chain.responses()) + # Second response was the tool-result turn. + second = chain._responses[1] + assert second.prompt.system == "be brief" + + def test_sync_chain_tool_result_turn_preserves_system_fragments( + self, mock_model + ): + tool_call = llm.ToolCall( + tool_call_id="c1", name="tick", arguments={} + ) + + class ChainMock(type(mock_model)): + def execute(self, prompt, stream, response, conversation): + if not self._queue: + yield "done" + return + msgs = self._queue.pop(0) + for m in msgs: + yield m + if not response._tool_calls: + response.add_tool_call(tool_call) + + def tick() -> str: + "Tick" + return "tock" + + m = ChainMock() + m.enqueue(["tool-turn"]) + m.enqueue(["final"]) + + chain = m.chain( + "q", + system="inline sys", + system_fragments=["fragment A", "fragment B"], + tools=[tick], + ) + list(chain.responses()) + second = chain._responses[1] + # prompt.system concatenates _system + system_fragments; all + # three strings should be preserved on the tool-result turn. + assert "inline sys" in second.prompt.system + assert "fragment A" in second.prompt.system + assert "fragment B" in second.prompt.system + + @pytest.mark.asyncio + async def test_async_chain_tool_result_turn_preserves_system( + self, async_mock_model + ): + tool_call = llm.ToolCall( + tool_call_id="c1", name="tick", arguments={} + ) + + class AsyncChainMock(type(async_mock_model)): + supports_tools = True + + async def execute(self, prompt, stream, response, conversation): + if not self._queue: + yield "done" + return + msgs = self._queue.pop(0) + for m in msgs: + yield m + if not response._tool_calls: + response.add_tool_call(tool_call) + + def tick() -> str: + "Tick" + return "tock" + + m = AsyncChainMock() + m.enqueue(["tool-turn"]) + m.enqueue(["final"]) + + chain = m.chain("q", system="be brief", tools=[tick]) + responses = [] + async for r in chain.responses(): + responses.append(r) + second = chain._responses[1] + assert second.prompt.system == "be brief" + + # -- chain() accepts messages= (parity with prompt()) ----------------- From 1c317ab3feaed0310f22a2490e5afaf8c1c4ad91 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 22 Apr 2026 00:08:41 +0000 Subject: [PATCH 031/258] Ran cog --- README.md | 1 + build/lib/llm/__init__.py | 515 +++ build/lib/llm/__main__.py | 4 + build/lib/llm/cli.py | 4094 +++++++++++++++++ build/lib/llm/default_plugins/__init__.py | 0 .../lib/llm/default_plugins/default_tools.py | 8 + .../lib/llm/default_plugins/openai_models.py | 1212 +++++ build/lib/llm/embeddings.py | 367 ++ build/lib/llm/embeddings_migrations.py | 89 + build/lib/llm/errors.py | 6 + build/lib/llm/hookspecs.py | 35 + build/lib/llm/migrations.py | 420 ++ build/lib/llm/models.py | 2966 ++++++++++++ build/lib/llm/parts.py | 340 ++ build/lib/llm/plugins.py | 50 + build/lib/llm/py.typed | 0 build/lib/llm/serialization.py | 182 + build/lib/llm/templates.py | 92 + build/lib/llm/tools.py | 37 + build/lib/llm/utils.py | 735 +++ docs/help.md | 2 + 21 files changed, 11155 insertions(+) create mode 100644 build/lib/llm/__init__.py create mode 100644 build/lib/llm/__main__.py create mode 100644 build/lib/llm/cli.py create mode 100644 build/lib/llm/default_plugins/__init__.py create mode 100644 build/lib/llm/default_plugins/default_tools.py create mode 100644 build/lib/llm/default_plugins/openai_models.py create mode 100644 build/lib/llm/embeddings.py create mode 100644 build/lib/llm/embeddings_migrations.py create mode 100644 build/lib/llm/errors.py create mode 100644 build/lib/llm/hookspecs.py create mode 100644 build/lib/llm/migrations.py create mode 100644 build/lib/llm/models.py create mode 100644 build/lib/llm/parts.py create mode 100644 build/lib/llm/plugins.py create mode 100644 build/lib/llm/py.typed create mode 100644 build/lib/llm/serialization.py create mode 100644 build/lib/llm/templates.py create mode 100644 build/lib/llm/tools.py create mode 100644 build/lib/llm/utils.py diff --git a/README.md b/README.md index 860446f06..526b926bb 100644 --- a/README.md +++ b/README.md @@ -302,6 +302,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Accessing the underlying JSON](https://llm.datasette.io/en/stable/python-api.html#accessing-the-underlying-json) * [Token usage](https://llm.datasette.io/en/stable/python-api.html#token-usage) * [Streaming responses](https://llm.datasette.io/en/stable/python-api.html#streaming-responses) + * [Structured messages and streaming events](https://llm.datasette.io/en/stable/python-api.html#structured-messages-and-streaming-events) * [Async models](https://llm.datasette.io/en/stable/python-api.html#async-models) * [`AsyncResponse`](https://llm.datasette.io/en/stable/python-api.html#llm.AsyncResponse) * [Tool functions can be sync or async](https://llm.datasette.io/en/stable/python-api.html#tool-functions-can-be-sync-or-async) diff --git a/build/lib/llm/__init__.py b/build/lib/llm/__init__.py new file mode 100644 index 000000000..bb84c3911 --- /dev/null +++ b/build/lib/llm/__init__.py @@ -0,0 +1,515 @@ +from .hookspecs import hookimpl +from .errors import ( + ModelError, + NeedsKeyException, +) +from .models import ( + AsyncConversation, + AsyncKeyModel, + AsyncModel, + AsyncResponse, + Attachment, + CancelToolCall, + Conversation, + EmbeddingModel, + EmbeddingModelWithAliases, + KeyModel, + Model, + ModelWithAliases, + Options, + Prompt, + Response, + Tool, + Toolbox, + ToolCall, + ToolOutput, + ToolResult, + Usage, +) +from .parts import ( + AttachmentPart, + Message, + Part, + ReasoningPart, + StreamEvent, + TextPart, + ToolCallPart, + ToolResultPart, + assistant, + system, + tool_message, + user, +) +from .utils import schema_dsl, Fragment +from .embeddings import Collection +from .templates import Template +from .plugins import pm, load_plugins +import click +from typing import Any, Dict, List, Optional, Callable, Type, Union +import inspect +import json +import os +import pathlib +import struct + +__all__ = [ + "AsyncConversation", + "AsyncKeyModel", + "AsyncModel", + "AsyncResponse", + "assistant", + "Attachment", + "AttachmentPart", + "CancelToolCall", + "Collection", + "Conversation", + "Fragment", + "get_async_model", + "get_key", + "get_model", + "hookimpl", + "KeyModel", + "Message", + "Model", + "ModelError", + "NeedsKeyException", + "Options", + "Part", + "Prompt", + "ReasoningPart", + "Response", + "schema_dsl", + "StreamEvent", + "system", + "Template", + "TextPart", + "Tool", + "Toolbox", + "ToolCall", + "ToolCallPart", + "tool_message", + "ToolOutput", + "ToolResult", + "ToolResultPart", + "Usage", + "user", + "user_dir", +] +DEFAULT_MODEL = "gpt-4o-mini" + + +def get_plugins(all=False): + plugins = [] + plugin_to_distinfo = dict(pm.list_plugin_distinfo()) + for plugin in pm.get_plugins(): + if not all and plugin.__name__.startswith("llm.default_plugins."): + continue + plugin_info = { + "name": plugin.__name__, + "hooks": [h.name for h in pm.get_hookcallers(plugin)], + } + distinfo = plugin_to_distinfo.get(plugin) + if distinfo: + plugin_info["version"] = distinfo.version + plugin_info["name"] = ( + getattr(distinfo, "name", None) or distinfo.project_name + ) + plugins.append(plugin_info) + return plugins + + +def get_models_with_aliases() -> List["ModelWithAliases"]: + model_aliases = [] + + # Include aliases from aliases.json + aliases_path = user_dir() / "aliases.json" + extra_model_aliases: Dict[str, list] = {} + if aliases_path.exists(): + configured_aliases = json.loads(aliases_path.read_text()) + for alias, model_id in configured_aliases.items(): + extra_model_aliases.setdefault(model_id, []).append(alias) + + def register(model, async_model=None, aliases=None): + alias_list = list(aliases or []) + if model.model_id in extra_model_aliases: + alias_list.extend(extra_model_aliases[model.model_id]) + model_aliases.append(ModelWithAliases(model, async_model, alias_list)) + + load_plugins() + pm.hook.register_models(register=register, model_aliases=model_aliases) + + return model_aliases + + +def _get_loaders(hook_method) -> Dict[str, Callable]: + load_plugins() + loaders = {} + + def register(prefix, loader): + suffix = 0 + prefix_to_try = prefix + while prefix_to_try in loaders: + suffix += 1 + prefix_to_try = f"{prefix}_{suffix}" + loaders[prefix_to_try] = loader + + hook_method(register=register) + return loaders + + +def get_template_loaders() -> Dict[str, Callable[[str], Template]]: + """Get template loaders registered by plugins.""" + return _get_loaders(pm.hook.register_template_loaders) + + +def get_fragment_loaders() -> Dict[ + str, + Callable[[str], Union[Fragment, Attachment, List[Union[Fragment, Attachment]]]], +]: + """Get fragment loaders registered by plugins.""" + return _get_loaders(pm.hook.register_fragment_loaders) + + +def get_tools() -> Dict[str, Union[Tool, Type[Toolbox]]]: + """Return all tools (llm.Tool and llm.Toolbox) registered by plugins.""" + load_plugins() + tools: Dict[str, Union[Tool, Type[Toolbox]]] = {} + + # Variable to track current plugin name + current_plugin_name = None + + def register( + tool_or_function: Union[Tool, Type[Toolbox], Callable[..., Any]], + name: Optional[str] = None, + ) -> None: + tool: Union[Tool, Type[Toolbox], None] = None + + # If it's a Toolbox class, set the plugin field on it + if inspect.isclass(tool_or_function): + if issubclass(tool_or_function, Toolbox): + tool = tool_or_function + if current_plugin_name: + tool.plugin = current_plugin_name + tool.name = name or tool.__name__ + else: + raise TypeError( + "Toolbox classes must inherit from llm.Toolbox, {} does not.".format( + tool_or_function.__name__ + ) + ) + + # If it's already a Tool instance, use it directly + elif isinstance(tool_or_function, Tool): + tool = tool_or_function + if name: + tool.name = name + if current_plugin_name: + tool.plugin = current_plugin_name + + # If it's a bare function, wrap it in a Tool + else: + tool = Tool.function(tool_or_function, name=name) + if current_plugin_name: + tool.plugin = current_plugin_name + + # Get the name for the tool/toolbox + if tool: + # For Toolbox classes, use their name attribute or class name + if inspect.isclass(tool) and issubclass(tool, Toolbox): + prefix = name or getattr(tool, "name", tool.__name__) or "" + else: + prefix = name or tool.name or "" + + suffix = 0 + candidate = prefix + + # Avoid name collisions + while candidate in tools: + suffix += 1 + candidate = f"{prefix}_{suffix}" + + tools[candidate] = tool + + # Call each plugin's register_tools hook individually to track current_plugin_name + for plugin in pm.get_plugins(): + current_plugin_name = pm.get_name(plugin) + hook_caller = pm.hook.register_tools + plugin_impls = [ + impl for impl in hook_caller.get_hookimpls() if impl.plugin is plugin + ] + for impl in plugin_impls: + impl.function(register=register) + + return tools + + +def get_embedding_models_with_aliases() -> List["EmbeddingModelWithAliases"]: + model_aliases = [] + + # Include aliases from aliases.json + aliases_path = user_dir() / "aliases.json" + extra_model_aliases: Dict[str, list] = {} + if aliases_path.exists(): + configured_aliases = json.loads(aliases_path.read_text()) + for alias, model_id in configured_aliases.items(): + extra_model_aliases.setdefault(model_id, []).append(alias) + + def register(model, aliases=None): + alias_list = list(aliases or []) + if model.model_id in extra_model_aliases: + alias_list.extend(extra_model_aliases[model.model_id]) + model_aliases.append(EmbeddingModelWithAliases(model, alias_list)) + + load_plugins() + pm.hook.register_embedding_models(register=register) + + return model_aliases + + +def get_embedding_models(): + models = [] + + def register(model, aliases=None): + models.append(model) + + load_plugins() + pm.hook.register_embedding_models(register=register) + return models + + +def get_embedding_model(name): + aliases = get_embedding_model_aliases() + try: + return aliases[name] + except KeyError: + raise UnknownModelError("Unknown model: " + str(name)) + + +def get_embedding_model_aliases() -> Dict[str, EmbeddingModel]: + model_aliases = {} + for model_with_aliases in get_embedding_models_with_aliases(): + for alias in model_with_aliases.aliases: + model_aliases[alias] = model_with_aliases.model + model_aliases[model_with_aliases.model.model_id] = model_with_aliases.model + return model_aliases + + +def get_async_model_aliases() -> Dict[str, AsyncModel]: + async_model_aliases = {} + for model_with_aliases in get_models_with_aliases(): + if model_with_aliases.async_model: + for alias in model_with_aliases.aliases: + async_model_aliases[alias] = model_with_aliases.async_model + async_model_aliases[model_with_aliases.model.model_id] = ( + model_with_aliases.async_model + ) + return async_model_aliases + + +def get_model_aliases() -> Dict[str, Model]: + model_aliases = {} + for model_with_aliases in get_models_with_aliases(): + if model_with_aliases.model: + for alias in model_with_aliases.aliases: + model_aliases[alias] = model_with_aliases.model + model_aliases[model_with_aliases.model.model_id] = model_with_aliases.model + return model_aliases + + +class UnknownModelError(KeyError): + pass + + +def get_models() -> List[Model]: + "Get all registered models" + models_with_aliases = get_models_with_aliases() + return [mwa.model for mwa in models_with_aliases if mwa.model] + + +def get_async_models() -> List[AsyncModel]: + "Get all registered async models" + models_with_aliases = get_models_with_aliases() + return [mwa.async_model for mwa in models_with_aliases if mwa.async_model] + + +def get_async_model(name: Optional[str] = None) -> AsyncModel: + "Get an async model by name or alias" + aliases = get_async_model_aliases() + name = name or get_default_model() + try: + return aliases[name] + except KeyError: + # Does a sync model exist? + sync_model = None + try: + sync_model = get_model(name, _skip_async=True) + except UnknownModelError: + pass + if sync_model: + raise UnknownModelError("Unknown async model (sync model exists): " + name) + else: + raise UnknownModelError("Unknown model: " + name) + + +def get_model(name: Optional[str] = None, _skip_async: bool = False) -> Model: + "Get a model by name or alias" + aliases = get_model_aliases() + name = name or get_default_model() + try: + return aliases[name] + except KeyError: + # Does an async model exist? + if _skip_async: + raise UnknownModelError("Unknown model: " + name) + async_model = None + try: + async_model = get_async_model(name) + except UnknownModelError: + pass + if async_model: + raise UnknownModelError("Unknown model (async model exists): " + name) + else: + raise UnknownModelError("Unknown model: " + name) + + +def get_key( + explicit_key: Optional[str] = None, + key_alias: Optional[str] = None, + env_var: Optional[str] = None, + *, + alias: Optional[str] = None, + env: Optional[str] = None, + input: Optional[str] = None, +) -> Optional[str]: + """ + Return an API key based on a hierarchy of potential sources. You should use the keyword arguments, + the positional arguments are here purely for backwards-compatibility with older code. + + :param input: Input provided by the user. This may be the key, or an alias of a key in keys.json. + :param alias: The alias used to retrieve the key from the keys.json file. + :param env: Name of the environment variable to check for the key as a final fallback. + """ + if alias: + key_alias = alias + if env: + env_var = env + if input: + explicit_key = input + stored_keys = load_keys() + # If user specified an alias, use the key stored for that alias + if explicit_key in stored_keys: + return stored_keys[explicit_key] + if explicit_key: + # User specified a key that's not an alias, use that + return explicit_key + # Stored key over-rides environment variables over-ride the default key + if key_alias in stored_keys: + return stored_keys[key_alias] + # Finally try environment variable + if env_var and os.environ.get(env_var): + return os.environ[env_var] + # Couldn't find it + return None + + +def load_keys(): + path = user_dir() / "keys.json" + if path.exists(): + return json.loads(path.read_text()) + else: + return {} + + +def user_dir(): + llm_user_path = os.environ.get("LLM_USER_PATH") + if llm_user_path: + path = pathlib.Path(llm_user_path) + else: + path = pathlib.Path(click.get_app_dir("io.datasette.llm")) + path.mkdir(exist_ok=True, parents=True) + return path + + +def set_alias(alias, model_id_or_alias): + """ + Set an alias to point to the specified model. + """ + path = user_dir() / "aliases.json" + path.parent.mkdir(parents=True, exist_ok=True) + if not path.exists(): + path.write_text("{}\n") + try: + current = json.loads(path.read_text()) + except json.decoder.JSONDecodeError: + # We're going to write a valid JSON file in a moment: + current = {} + # Resolve model_id_or_alias to a model_id + try: + model = get_model(model_id_or_alias) + model_id = model.model_id + except UnknownModelError: + # Try to resolve it to an embedding model + try: + model = get_embedding_model(model_id_or_alias) + model_id = model.model_id + except UnknownModelError: + # Set the alias to the exact string they provided instead + model_id = model_id_or_alias + current[alias] = model_id + path.write_text(json.dumps(current, indent=4) + "\n") + + +def remove_alias(alias): + """ + Remove an alias. + """ + path = user_dir() / "aliases.json" + if not path.exists(): + raise KeyError("No aliases.json file exists") + try: + current = json.loads(path.read_text()) + except json.decoder.JSONDecodeError: + raise KeyError("aliases.json file is not valid JSON") + if alias not in current: + raise KeyError("No such alias: {}".format(alias)) + del current[alias] + path.write_text(json.dumps(current, indent=4) + "\n") + + +def encode(values): + return struct.pack("<" + "f" * len(values), *values) + + +def decode(binary): + return struct.unpack("<" + "f" * (len(binary) // 4), binary) + + +def cosine_similarity(a, b): + dot_product = sum(x * y for x, y in zip(a, b)) + magnitude_a = sum(x * x for x in a) ** 0.5 + magnitude_b = sum(x * x for x in b) ** 0.5 + return dot_product / (magnitude_a * magnitude_b) + + +def get_default_model(filename="default_model.txt", default=DEFAULT_MODEL): + path = user_dir() / filename + if path.exists(): + return path.read_text().strip() + else: + return default + + +def set_default_model(model, filename="default_model.txt"): + path = user_dir() / filename + if model is None and path.exists(): + path.unlink() + else: + path.write_text(model) + + +def get_default_embedding_model(): + return get_default_model("default_embedding_model.txt", None) + + +def set_default_embedding_model(model): + set_default_model(model, "default_embedding_model.txt") diff --git a/build/lib/llm/__main__.py b/build/lib/llm/__main__.py new file mode 100644 index 000000000..98dcca0c2 --- /dev/null +++ b/build/lib/llm/__main__.py @@ -0,0 +1,4 @@ +from .cli import cli + +if __name__ == "__main__": + cli() diff --git a/build/lib/llm/cli.py b/build/lib/llm/cli.py new file mode 100644 index 000000000..f5cdd586d --- /dev/null +++ b/build/lib/llm/cli.py @@ -0,0 +1,4094 @@ +import asyncio +import click +from click_default_group import DefaultGroup +from dataclasses import asdict +from importlib.metadata import version +import io +import json +import os +from llm import ( + Attachment, + AsyncConversation, + AsyncKeyModel, + AsyncResponse, + CancelToolCall, + Collection, + Conversation, + Fragment, + Response, + Template, + Tool, + Toolbox, + UnknownModelError, + KeyModel, + encode, + get_async_model, + get_default_model, + get_default_embedding_model, + get_embedding_models_with_aliases, + get_embedding_model_aliases, + get_embedding_model, + get_plugins, + get_tools, + get_fragment_loaders, + get_template_loaders, + get_model, + get_model_aliases, + get_models_with_aliases, + user_dir, + set_alias, + set_default_model, + set_default_embedding_model, + remove_alias, +) +from llm.models import _BaseConversation, ChainResponse + +from .migrations import migrate +from .plugins import pm, load_plugins +from .utils import ( + ensure_fragment, + extract_fenced_code_block, + find_unused_key, + has_plugin_prefix, + instantiate_from_spec, + make_schema_id, + maybe_fenced_code, + mimetype_from_path, + mimetype_from_string, + multi_schema, + output_rows_as_json, + resolve_schema_input, + schema_dsl, + schema_summary, + token_usage_string, + truncate_string, +) +import base64 +import httpx +import inspect +import pathlib +import pydantic +import re +import readline +from runpy import run_module +import shutil +import sqlite_utils +from sqlite_utils.utils import rows_from_file, Format +import sys +import textwrap +from typing import cast, Dict, Optional, Iterable, List, Union, Tuple, Type, Any +import warnings +import yaml + +warnings.simplefilter("ignore", ResourceWarning) + +DEFAULT_TEMPLATE = "prompt: " + + +class FragmentNotFound(Exception): + pass + + +def display_stream_events(events, *, show_reasoning=True): + """Consume a sync iterator of StreamEvents and write them. + + Text events go to stdout. Reasoning events go to stderr in dim style. + A newline is written to stderr at each reasoning→text transition so + the assistant text starts on a fresh visual line. + """ + was_reasoning = False + for event in events: + if event.type == "text": + if was_reasoning and show_reasoning: + click.echo("", err=True) + was_reasoning = False + click.echo(event.chunk, nl=False) + elif event.type == "reasoning" and show_reasoning: + was_reasoning = True + click.echo(click.style(event.chunk, dim=True), nl=False, err=True) + + +async def display_async_stream_events(events, *, show_reasoning=True): + """Async counterpart of display_stream_events.""" + was_reasoning = False + async for event in events: + if event.type == "text": + if was_reasoning and show_reasoning: + click.echo("", err=True) + was_reasoning = False + click.echo(event.chunk, nl=False) + elif event.type == "reasoning" and show_reasoning: + was_reasoning = True + click.echo(click.style(event.chunk, dim=True), nl=False, err=True) + + +def validate_fragment_alias(ctx, param, value): + if not re.match(r"^[a-zA-Z0-9_-]+$", value): + raise click.BadParameter("Fragment alias must be alphanumeric") + return value + + +def resolve_fragments( + db: sqlite_utils.Database, fragments: Iterable[str], allow_attachments: bool = False +) -> List[Union[Fragment, Attachment]]: + """ + Resolve fragment strings into a mixed of llm.Fragment() and llm.Attachment() objects. + """ + + def _load_by_alias(fragment: str) -> Tuple[Optional[str], Optional[str]]: + rows = list( + db.query( + """ + select content, source from fragments + left join fragment_aliases on fragments.id = fragment_aliases.fragment_id + where alias = :alias or hash = :alias limit 1 + """, + {"alias": fragment}, + ) + ) + if rows: + row = rows[0] + return row["content"], row["source"] + return None, None + + # The fragment strings could be URLs or paths or plugin references + resolved: List[Union[Fragment, Attachment]] = [] + for fragment in fragments: + if fragment.startswith("http://") or fragment.startswith("https://"): + llm_version = version("llm") + headers = {"User-Agent": f"llm/{llm_version} (https://llm.datasette.io/)"} + client = httpx.Client( + follow_redirects=True, max_redirects=3, headers=headers + ) + response = client.get(fragment) + response.raise_for_status() + resolved.append(Fragment(response.text, fragment)) + elif fragment == "-": + resolved.append(Fragment(sys.stdin.read(), "-")) + elif has_plugin_prefix(fragment): + prefix, rest = fragment.split(":", 1) + loaders = get_fragment_loaders() + if prefix not in loaders: + raise FragmentNotFound("Unknown fragment prefix: {}".format(prefix)) + loader = loaders[prefix] + try: + result = loader(rest) + if not isinstance(result, list): + result = [result] + if not allow_attachments and any( + isinstance(r, Attachment) for r in result + ): + raise FragmentNotFound( + "Fragment loader {} returned a disallowed attachment".format( + prefix + ) + ) + resolved.extend(result) + except Exception as ex: + raise FragmentNotFound( + "Could not load fragment {}: {}".format(fragment, ex) + ) + else: + # Try from the DB + content, source = _load_by_alias(fragment) + if content is not None: + resolved.append(Fragment(content, source)) + else: + # Now try path + path = pathlib.Path(fragment) + if path.exists(): + resolved.append(Fragment(path.read_text(), str(path.resolve()))) + else: + raise FragmentNotFound(f"Fragment '{fragment}' not found") + return resolved + + +def process_fragments_in_chat( + db: sqlite_utils.Database, prompt: str +) -> tuple[str, list[Fragment], list[Attachment]]: + """ + Process any !fragment commands in a chat prompt and return the modified prompt plus resolved fragments and attachments. + """ + prompt_lines = [] + fragments = [] + attachments = [] + for line in prompt.splitlines(): + if line.startswith("!fragment "): + try: + fragment_strs = line.strip().removeprefix("!fragment ").split() + fragments_and_attachments = resolve_fragments( + db, fragments=fragment_strs, allow_attachments=True + ) + fragments += [ + fragment + for fragment in fragments_and_attachments + if isinstance(fragment, Fragment) + ] + attachments += [ + attachment + for attachment in fragments_and_attachments + if isinstance(attachment, Attachment) + ] + except FragmentNotFound as ex: + raise click.ClickException(str(ex)) + else: + prompt_lines.append(line) + return "\n".join(prompt_lines), fragments, attachments + + +class AttachmentError(Exception): + """Exception raised for errors in attachment resolution.""" + + pass + + +def resolve_attachment(value): + """ + Resolve an attachment from a string value which could be: + - "-" for stdin + - A URL + - A file path + + Returns an Attachment object. + Raises AttachmentError if the attachment cannot be resolved. + """ + if value == "-": + content = sys.stdin.buffer.read() + # Try to guess type + mimetype = mimetype_from_string(content) + if mimetype is None: + raise AttachmentError("Could not determine mimetype of stdin") + return Attachment(type=mimetype, path=None, url=None, content=content) + + if "://" in value: + # Confirm URL exists and try to guess type + try: + response = httpx.head(value) + response.raise_for_status() + mimetype = response.headers.get("content-type") + except httpx.HTTPError as ex: + raise AttachmentError(str(ex)) + return Attachment(type=mimetype, path=None, url=value, content=None) + + # Check that the file exists + path = pathlib.Path(value) + if not path.exists(): + raise AttachmentError(f"File {value} does not exist") + path = path.resolve() + + # Try to guess type + mimetype = mimetype_from_path(str(path)) + if mimetype is None: + raise AttachmentError(f"Could not determine mimetype of {value}") + + return Attachment(type=mimetype, path=str(path), url=None, content=None) + + +class AttachmentType(click.ParamType): + name = "attachment" + + def convert(self, value, param, ctx): + try: + return resolve_attachment(value) + except AttachmentError as e: + self.fail(str(e), param, ctx) + + +def resolve_attachment_with_type(value: str, mimetype: str) -> Attachment: + if "://" in value: + attachment = Attachment(mimetype, None, value, None) + elif value == "-": + content = sys.stdin.buffer.read() + attachment = Attachment(mimetype, None, None, content) + else: + # Look for file + path = pathlib.Path(value) + if not path.exists(): + raise click.BadParameter(f"File {value} does not exist") + path = path.resolve() + attachment = Attachment(mimetype, str(path), None, None) + return attachment + + +def attachment_types_callback(ctx, param, values) -> List[Attachment]: + collected = [] + for value, mimetype in values: + collected.append(resolve_attachment_with_type(value, mimetype)) + return collected + + +def json_validator(object_name): + def validator(ctx, param, value): + if value is None: + return value + try: + obj = json.loads(value) + if not isinstance(obj, dict): + raise click.BadParameter(f"{object_name} must be a JSON object") + return obj + except json.JSONDecodeError: + raise click.BadParameter(f"{object_name} must be valid JSON") + + return validator + + +def schema_option(fn): + click.option( + "schema_input", + "--schema", + help="JSON schema, filepath or ID", + )(fn) + return fn + + +@click.group( + cls=DefaultGroup, + default="prompt", + default_if_no_args=True, + context_settings={"help_option_names": ["-h", "--help"]}, +) +@click.version_option() +def cli(): + """ + Access Large Language Models from the command-line + + Documentation: https://llm.datasette.io/ + + LLM can run models from many different providers. Consult the + plugin directory for a list of available models: + + https://llm.datasette.io/en/stable/plugins/directory.html + + To get started with OpenAI, obtain an API key from them and: + + \b + $ llm keys set openai + Enter key: ... + + Then execute a prompt like this: + + llm 'Five outrageous names for a pet pelican' + + For a full list of prompting options run: + + llm prompt --help + """ + + +@cli.command(name="prompt") +@click.argument("prompt", required=False) +@click.option("-s", "--system", help="System prompt to use") +@click.option("model_id", "-m", "--model", help="Model to use", envvar="LLM_MODEL") +@click.option( + "-d", + "--database", + type=click.Path(readable=True, dir_okay=False), + help="Path to log database", +) +@click.option( + "queries", + "-q", + "--query", + multiple=True, + help="Use first model matching these strings", +) +@click.option( + "attachments", + "-a", + "--attachment", + type=AttachmentType(), + multiple=True, + help="Attachment path or URL or -", +) +@click.option( + "attachment_types", + "--at", + "--attachment-type", + type=(str, str), + multiple=True, + callback=attachment_types_callback, + help="\b\nAttachment with explicit mimetype,\n--at image.jpg image/jpeg", +) +@click.option( + "tools", + "-T", + "--tool", + multiple=True, + help="Name of a tool to make available to the model", +) +@click.option( + "python_tools", + "--functions", + help="Python code block or file path defining functions to register as tools", + multiple=True, +) +@click.option( + "tools_debug", + "--td", + "--tools-debug", + is_flag=True, + help="Show full details of tool executions", + envvar="LLM_TOOLS_DEBUG", +) +@click.option( + "tools_approve", + "--ta", + "--tools-approve", + is_flag=True, + help="Manually approve every tool execution", +) +@click.option( + "chain_limit", + "--cl", + "--chain-limit", + type=int, + default=5, + help="How many chained tool responses to allow, default 5, set 0 for unlimited", +) +@click.option( + "options", + "-o", + "--option", + type=(str, str), + multiple=True, + help="key/value options for the model", +) +@schema_option +@click.option( + "--schema-multi", + help="JSON schema to use for multiple results", +) +@click.option( + "fragments", + "-f", + "--fragment", + multiple=True, + help="Fragment (alias, URL, hash or file path) to add to the prompt", +) +@click.option( + "system_fragments", + "--sf", + "--system-fragment", + multiple=True, + help="Fragment to add to system prompt", +) +@click.option("-t", "--template", help="Template to use") +@click.option( + "-p", + "--param", + multiple=True, + type=(str, str), + help="Parameters for template", +) +@click.option("--no-stream", is_flag=True, help="Do not stream output") +@click.option("-n", "--no-log", is_flag=True, help="Don't log to database") +@click.option("--log", is_flag=True, help="Log prompt and response to the database") +@click.option( + "-R", "--no-reasoning", is_flag=True, help="Don't display reasoning output" +) +@click.option( + "_continue", + "-c", + "--continue", + is_flag=True, + flag_value=-1, + help="Continue the most recent conversation.", +) +@click.option( + "conversation_id", + "--cid", + "--conversation", + help="Continue the conversation with the given ID.", +) +@click.option("--key", help="API key to use") +@click.option("--save", help="Save prompt with this template name") +@click.option("async_", "--async", is_flag=True, help="Run prompt asynchronously") +@click.option("-u", "--usage", is_flag=True, help="Show token usage") +@click.option("-x", "--extract", is_flag=True, help="Extract first fenced code block") +@click.option( + "extract_last", + "--xl", + "--extract-last", + is_flag=True, + help="Extract last fenced code block", +) +def prompt( + prompt, + system, + model_id, + database, + queries, + attachments, + attachment_types, + tools, + python_tools, + tools_debug, + tools_approve, + chain_limit, + options, + schema_input, + schema_multi, + fragments, + system_fragments, + template, + param, + no_stream, + no_log, + log, + no_reasoning, + _continue, + conversation_id, + key, + save, + async_, + usage, + extract, + extract_last, +): + """ + Execute a prompt + + Documentation: https://llm.datasette.io/en/stable/usage.html + + Examples: + + \b + llm 'Capital of France?' + llm 'Capital of France?' -m gpt-4o + llm 'Capital of France?' -s 'answer in Spanish' + + Multi-modal models can be called with attachments like this: + + \b + llm 'Extract text from this image' -a image.jpg + llm 'Describe' -a https://static.simonwillison.net/static/2024/pelicans.jpg + cat image | llm 'describe image' -a - + # With an explicit mimetype: + cat image | llm 'describe image' --at - image/jpeg + + The -x/--extract option returns just the content of the first ``` fenced code + block, if one is present. If none are present it returns the full response. + + \b + llm 'JavaScript function for reversing a string' -x + """ + if log and no_log: + raise click.ClickException("--log and --no-log are mutually exclusive") + + log_path = pathlib.Path(database) if database else logs_db_path() + (log_path.parent).mkdir(parents=True, exist_ok=True) + db = sqlite_utils.Database(log_path) + migrate(db) + + if queries and not model_id: + # Use -q options to find model with shortest model_id + matches = [] + for model_with_aliases in get_models_with_aliases(): + if all(model_with_aliases.matches(q) for q in queries): + matches.append(model_with_aliases.model.model_id) + if not matches: + raise click.ClickException( + "No model found matching queries {}".format(", ".join(queries)) + ) + model_id = min(matches, key=len) + + if schema_multi: + schema_input = schema_multi + + schema = resolve_schema_input(db, schema_input, load_template) + + if schema_multi: + # Convert that schema into multiple "items" of the same schema + schema = multi_schema(schema) + + def read_prompt(): + nonlocal prompt, schema + + # Is there extra prompt available on stdin? + stdin_prompt = None + if not sys.stdin.isatty(): + stdin_prompt = sys.stdin.read() + + if stdin_prompt: + bits = [stdin_prompt] + if prompt: + bits.append(prompt) + prompt = " ".join(bits) + + if ( + prompt is None + and not save + and sys.stdin.isatty() + and not attachments + and not attachment_types + and not schema + and not fragments + ): + # Hang waiting for input to stdin (unless --save) + prompt = sys.stdin.read() + return prompt + + if save: + # We are saving their prompt/system/etc to a new template + # Fields to save: prompt, system, model - and more in the future + disallowed_options = [] + for option, var in ( + ("--template", template), + ("--continue", _continue), + ("--cid", conversation_id), + ): + if var: + disallowed_options.append(option) + if disallowed_options: + raise click.ClickException( + "--save cannot be used with {}".format(", ".join(disallowed_options)) + ) + path = template_dir() / f"{save}.yaml" + to_save = {} + if model_id: + model_aliases = get_model_aliases() + try: + to_save["model"] = model_aliases[model_id].model_id + except KeyError: + raise click.ClickException("'{}' is not a known model".format(model_id)) + prompt = read_prompt() + if prompt: + to_save["prompt"] = prompt + if system: + to_save["system"] = system + if param: + to_save["defaults"] = dict(param) + if extract: + to_save["extract"] = True + if extract_last: + to_save["extract_last"] = True + if schema: + to_save["schema_object"] = schema + if fragments: + to_save["fragments"] = list(fragments) + if system_fragments: + to_save["system_fragments"] = list(system_fragments) + if python_tools: + to_save["functions"] = "\n\n".join(python_tools) + if tools: + to_save["tools"] = list(tools) + if attachments: + # Only works for attachments with a path or url + to_save["attachments"] = [ + (a.path or a.url) for a in attachments if (a.path or a.url) + ] + if attachment_types: + to_save["attachment_types"] = [ + {"type": a.type, "value": a.path or a.url} + for a in attachment_types + if (a.path or a.url) + ] + if options: + # Need to validate and convert their types first + model = get_model(model_id or get_default_model()) + try: + options_model = model.Options(**dict(options)) + # Use model_dump(mode="json") so Enums become their .value strings + to_save["options"] = { + k: v + for k, v in options_model.model_dump(mode="json").items() + if v is not None + } + except pydantic.ValidationError as ex: + raise click.ClickException(render_errors(ex.errors())) + path.write_text( + yaml.safe_dump( + to_save, + indent=4, + default_flow_style=False, + sort_keys=False, + ), + "utf-8", + ) + return + + if template: + params = dict(param) + # Cannot be used with system + try: + template_obj = load_template(template) + except LoadTemplateError as ex: + raise click.ClickException(str(ex)) + if not (extract or extract_last): + extract = template_obj.extract + extract_last = template_obj.extract_last + # Combine with template fragments/system_fragments + if template_obj.fragments: + fragments = [*template_obj.fragments, *fragments] + if template_obj.system_fragments: + system_fragments = [*template_obj.system_fragments, *system_fragments] + if template_obj.schema_object: + schema = template_obj.schema_object + if template_obj.tools: + tools = [*template_obj.tools, *tools] + if template_obj.functions and template_obj._functions_is_trusted: + python_tools = [template_obj.functions, *python_tools] + input_ = "" + if template_obj.options: + # Make options mutable (they start as a tuple) + options = list(options) + # Load any options, provided they were not set using -o already + specified_options = dict(options) + for option_name, option_value in template_obj.options.items(): + if option_name not in specified_options: + options.append((option_name, option_value)) + if "input" in template_obj.vars(): + input_ = read_prompt() + try: + template_prompt, template_system = template_obj.evaluate(input_, params) + if template_prompt: + # Combine with user prompt + if prompt and "input" not in template_obj.vars(): + prompt = template_prompt + "\n" + prompt + else: + prompt = template_prompt + if template_system and not system: + system = template_system + except Template.MissingVariables as ex: + raise click.ClickException(str(ex)) + if model_id is None and template_obj.model: + model_id = template_obj.model + # Merge in any attachments + if template_obj.attachments: + attachments = [ + resolve_attachment(a) for a in template_obj.attachments + ] + list(attachments) + if template_obj.attachment_types: + attachment_types = [ + resolve_attachment_with_type(at.value, at.type) + for at in template_obj.attachment_types + ] + list(attachment_types) + if extract or extract_last: + no_stream = True + + conversation = None + if conversation_id or _continue: + # Load the conversation - loads most recent if no ID provided + try: + conversation = load_conversation( + conversation_id, async_=async_, database=database + ) + except UnknownModelError as ex: + raise click.ClickException(str(ex)) + + if conversation_tools := _get_conversation_tools(conversation, tools): + tools = conversation_tools + + # Figure out which model we are using + if model_id is None: + if conversation: + model_id = conversation.model.model_id + else: + model_id = get_default_model() + + # Now resolve the model + try: + if async_: + model = get_async_model(model_id) + else: + model = get_model(model_id) + except UnknownModelError as ex: + raise click.ClickException(ex) + + if conversation is None and (tools or python_tools): + conversation = model.conversation() + + if conversation: + # To ensure it can see the key + conversation.model = model + + # Validate options + validated_options = {} + if options: + # Validate with pydantic + try: + validated_options = dict( + (key, value) + for key, value in model.Options(**dict(options)) + if value is not None + ) + except pydantic.ValidationError as ex: + raise click.ClickException(render_errors(ex.errors())) + + # Add on any default model options + default_options = get_model_options(model.model_id) + for key_, value in default_options.items(): + if key_ not in validated_options: + validated_options[key_] = value + + kwargs = {} + + resolved_attachments = [*attachments, *attachment_types] + + should_stream = model.can_stream and not no_stream + if not should_stream: + kwargs["stream"] = False + + if isinstance(model, (KeyModel, AsyncKeyModel)): + kwargs["key"] = key + + prompt = read_prompt() + response = None + + try: + fragments_and_attachments = resolve_fragments( + db, fragments, allow_attachments=True + ) + resolved_fragments = [ + fragment + for fragment in fragments_and_attachments + if isinstance(fragment, Fragment) + ] + resolved_attachments.extend( + attachment + for attachment in fragments_and_attachments + if isinstance(attachment, Attachment) + ) + resolved_system_fragments = resolve_fragments(db, system_fragments) + except FragmentNotFound as ex: + raise click.ClickException(str(ex)) + + prompt_method = model.prompt + if conversation: + prompt_method = conversation.prompt + + tool_implementations = _gather_tools(tools, python_tools) + + if tool_implementations: + prompt_method = conversation.chain + kwargs["options"] = validated_options + kwargs["chain_limit"] = chain_limit + if tools_debug: + kwargs["after_call"] = _debug_tool_call + if tools_approve: + kwargs["before_call"] = _approve_tool_call + kwargs["tools"] = tool_implementations + else: + # Merge in options for the .prompt() methods + kwargs.update(validated_options) + + try: + if async_: + + async def inner(): + if should_stream: + response = prompt_method( + prompt, + attachments=resolved_attachments, + system=system, + schema=schema, + fragments=resolved_fragments, + system_fragments=resolved_system_fragments, + **kwargs, + ) + await display_async_stream_events( + response.astream_events(), + show_reasoning=not no_reasoning, + ) + print("") + else: + response = prompt_method( + prompt, + fragments=resolved_fragments, + attachments=resolved_attachments, + schema=schema, + system=system, + system_fragments=resolved_system_fragments, + **kwargs, + ) + text = await response.text() + if extract or extract_last: + text = ( + extract_fenced_code_block(text, last=extract_last) or text + ) + print(text) + return response + + response = asyncio.run(inner()) + else: + response = prompt_method( + prompt, + fragments=resolved_fragments, + attachments=resolved_attachments, + system=system, + schema=schema, + system_fragments=resolved_system_fragments, + **kwargs, + ) + if should_stream: + display_stream_events( + response.stream_events(), + show_reasoning=not no_reasoning, + ) + print("") + else: + text = response.text() + if extract or extract_last: + text = extract_fenced_code_block(text, last=extract_last) or text + print(text) + # List of exceptions that should never be raised in pytest: + except (ValueError, NotImplementedError) as ex: + raise click.ClickException(str(ex)) + except Exception as ex: + # All other exceptions should raise in pytest, show to user otherwise + if getattr(sys, "_called_from_test", False) or os.environ.get( + "LLM_RAISE_ERRORS", None + ): + raise + raise click.ClickException(str(ex)) + + if usage: + if isinstance(response, ChainResponse): + responses = response._responses + else: + responses = [response] + for response_object in responses: + # Show token usage to stderr in yellow + click.echo( + click.style( + "Token usage: {}".format(response_object.token_usage()), + fg="yellow", + bold=True, + ), + err=True, + ) + + # Log responses to the database + if (logs_on() or log) and not no_log: + # Could be Response, AsyncResponse, ChainResponse, AsyncChainResponse + if isinstance(response, AsyncResponse): + response = asyncio.run(response.to_sync_response()) + # At this point ALL forms should have a log_to_db() method that works: + response.log_to_db(db) + + +@cli.command() +@click.option("-s", "--system", help="System prompt to use") +@click.option("model_id", "-m", "--model", help="Model to use", envvar="LLM_MODEL") +@click.option( + "_continue", + "-c", + "--continue", + is_flag=True, + flag_value=-1, + help="Continue the most recent conversation.", +) +@click.option( + "conversation_id", + "--cid", + "--conversation", + help="Continue the conversation with the given ID.", +) +@click.option( + "fragments", + "-f", + "--fragment", + multiple=True, + help="Fragment (alias, URL, hash or file path) to add to the prompt", +) +@click.option( + "system_fragments", + "--sf", + "--system-fragment", + multiple=True, + help="Fragment to add to system prompt", +) +@click.option("-t", "--template", help="Template to use") +@click.option( + "-p", + "--param", + multiple=True, + type=(str, str), + help="Parameters for template", +) +@click.option( + "options", + "-o", + "--option", + type=(str, str), + multiple=True, + help="key/value options for the model", +) +@click.option( + "-d", + "--database", + type=click.Path(readable=True, dir_okay=False), + help="Path to log database", +) +@click.option("--no-stream", is_flag=True, help="Do not stream output") +@click.option( + "-R", "--no-reasoning", is_flag=True, help="Don't display reasoning output" +) +@click.option("--key", help="API key to use") +@click.option( + "tools", + "-T", + "--tool", + multiple=True, + help="Name of a tool to make available to the model", +) +@click.option( + "python_tools", + "--functions", + help="Python code block or file path defining functions to register as tools", + multiple=True, +) +@click.option( + "tools_debug", + "--td", + "--tools-debug", + is_flag=True, + help="Show full details of tool executions", + envvar="LLM_TOOLS_DEBUG", +) +@click.option( + "tools_approve", + "--ta", + "--tools-approve", + is_flag=True, + help="Manually approve every tool execution", +) +@click.option( + "chain_limit", + "--cl", + "--chain-limit", + type=int, + default=5, + help="How many chained tool responses to allow, default 5, set 0 for unlimited", +) +def chat( + system, + model_id, + _continue, + conversation_id, + fragments, + system_fragments, + template, + param, + options, + no_stream, + no_reasoning, + key, + database, + tools, + python_tools, + tools_debug, + tools_approve, + chain_limit, +): + """ + Hold an ongoing chat with a model. + """ + # Left and right arrow keys to move cursor: + if sys.platform != "win32": + readline.parse_and_bind("\\e[D: backward-char") + readline.parse_and_bind("\\e[C: forward-char") + else: + readline.parse_and_bind("bind -x '\\e[D: backward-char'") + readline.parse_and_bind("bind -x '\\e[C: forward-char'") + log_path = pathlib.Path(database) if database else logs_db_path() + (log_path.parent).mkdir(parents=True, exist_ok=True) + db = sqlite_utils.Database(log_path) + migrate(db) + + conversation = None + if conversation_id or _continue: + # Load the conversation - loads most recent if no ID provided + try: + conversation = load_conversation(conversation_id, database=database) + except UnknownModelError as ex: + raise click.ClickException(str(ex)) + + if conversation_tools := _get_conversation_tools(conversation, tools): + tools = conversation_tools + + template_obj = None + if template: + params = dict(param) + try: + template_obj = load_template(template) + except LoadTemplateError as ex: + raise click.ClickException(str(ex)) + if model_id is None and template_obj.model: + model_id = template_obj.model + if template_obj.tools: + tools = [*template_obj.tools, *tools] + if template_obj.functions and template_obj._functions_is_trusted: + python_tools = [template_obj.functions, *python_tools] + + # Figure out which model we are using + if model_id is None: + if conversation: + model_id = conversation.model.model_id + else: + model_id = get_default_model() + + # Now resolve the model + try: + model = get_model(model_id) + except KeyError: + raise click.ClickException("'{}' is not a known model".format(model_id)) + + if conversation is None: + # Start a fresh conversation for this chat + conversation = Conversation(model=model) + else: + # Ensure it can see the API key + conversation.model = model + + if tools_debug: + conversation.after_call = _debug_tool_call + if tools_approve: + conversation.before_call = _approve_tool_call + + # Validate options + validated_options = get_model_options(model.model_id) + if options: + try: + validated_options = dict( + (key, value) + for key, value in model.Options(**dict(options)) + if value is not None + ) + except pydantic.ValidationError as ex: + raise click.ClickException(render_errors(ex.errors())) + + kwargs = {} + if validated_options: + kwargs["options"] = validated_options + + tool_functions = _gather_tools(tools, python_tools) + + if tool_functions: + kwargs["chain_limit"] = chain_limit + kwargs["tools"] = tool_functions + + should_stream = model.can_stream and not no_stream + if not should_stream: + kwargs["stream"] = False + + if key and isinstance(model, KeyModel): + kwargs["key"] = key + + try: + fragments_and_attachments = resolve_fragments( + db, fragments, allow_attachments=True + ) + argument_fragments = [ + fragment + for fragment in fragments_and_attachments + if isinstance(fragment, Fragment) + ] + argument_attachments = [ + attachment + for attachment in fragments_and_attachments + if isinstance(attachment, Attachment) + ] + argument_system_fragments = resolve_fragments(db, system_fragments) + except FragmentNotFound as ex: + raise click.ClickException(str(ex)) + + click.echo("Chatting with {}".format(model.model_id)) + click.echo("Type 'exit' or 'quit' to exit") + click.echo("Type '!multi' to enter multiple lines, then '!end' to finish") + click.echo("Type '!edit' to open your default editor and modify the prompt") + click.echo( + "Type '!fragment [ ...]' to insert one or more fragments" + ) + in_multi = False + + accumulated = [] + accumulated_fragments = [] + accumulated_attachments = [] + end_token = "!end" + while True: + prompt = click.prompt("", prompt_suffix="> " if not in_multi else "") + fragments = [] + attachments = [] + if argument_fragments: + fragments += argument_fragments + # fragments from --fragments will get added to the first message only + argument_fragments = [] + if argument_attachments: + attachments = argument_attachments + argument_attachments = [] + if prompt.strip().startswith("!multi"): + in_multi = True + bits = prompt.strip().split() + if len(bits) > 1: + end_token = "!end {}".format(" ".join(bits[1:])) + continue + if prompt.strip() == "!edit": + edited_prompt = click.edit() + if edited_prompt is None: + click.echo("Editor closed without saving.", err=True) + continue + prompt = edited_prompt.strip() + if prompt.strip().startswith("!fragment "): + prompt, fragments, attachments = process_fragments_in_chat(db, prompt) + + if in_multi: + if prompt.strip() == end_token: + prompt = "\n".join(accumulated) + fragments = accumulated_fragments + attachments = accumulated_attachments + in_multi = False + accumulated = [] + accumulated_fragments = [] + accumulated_attachments = [] + else: + if prompt: + accumulated.append(prompt) + accumulated_fragments += fragments + accumulated_attachments += attachments + continue + if template_obj: + try: + # Mirror prompt() logic: only pass input if template uses it + uses_input = "input" in template_obj.vars() + input_ = prompt if uses_input else "" + template_prompt, template_system = template_obj.evaluate(input_, params) + except Template.MissingVariables as ex: + raise click.ClickException(str(ex)) + if template_system and not system: + system = template_system + if template_prompt: + if prompt and not uses_input: + prompt = f"{template_prompt}\n{prompt}" + else: + prompt = template_prompt + if prompt.strip() in ("exit", "quit"): + break + + response = conversation.chain( + prompt, + fragments=fragments, + system_fragments=argument_system_fragments, + attachments=attachments, + system=system, + **kwargs, + ) + + # System prompt and system fragments only sent for the first message + system = None + argument_system_fragments = [] + display_stream_events( + response.stream_events(), + show_reasoning=not no_reasoning, + ) + response.log_to_db(db) + print("") + + +def load_conversation( + conversation_id: Optional[str], + async_=False, + database=None, +) -> Optional[_BaseConversation]: + log_path = pathlib.Path(database) if database else logs_db_path() + db = sqlite_utils.Database(log_path) + migrate(db) + if conversation_id is None: + # Return the most recent conversation, or None if there are none + matches = list(db["conversations"].rows_where(order_by="id desc", limit=1)) + if matches: + conversation_id = matches[0]["id"] + else: + return None + try: + row = cast(sqlite_utils.db.Table, db["conversations"]).get(conversation_id) + except sqlite_utils.db.NotFoundError: + raise click.ClickException( + "No conversation found with id={}".format(conversation_id) + ) + # Inflate that conversation + conversation_class = AsyncConversation if async_ else Conversation + response_class = AsyncResponse if async_ else Response + conversation = conversation_class.from_row(row) + for response in db["responses"].rows_where( + "conversation_id = ?", [conversation_id] + ): + conversation.responses.append(response_class.from_row(db, response)) + return conversation + + +@cli.group( + cls=DefaultGroup, + default="list", + default_if_no_args=True, +) +def keys(): + "Manage stored API keys for different models" + + +@keys.command(name="list") +def keys_list(): + "List names of all stored keys" + path = user_dir() / "keys.json" + if not path.exists(): + click.echo("No keys found") + return + keys = json.loads(path.read_text()) + for key in sorted(keys.keys()): + if key != "// Note": + click.echo(key) + + +@keys.command(name="path") +def keys_path_command(): + "Output the path to the keys.json file" + click.echo(user_dir() / "keys.json") + + +@keys.command(name="get") +@click.argument("name") +def keys_get(name): + """ + Return the value of a stored key + + Example usage: + + \b + export OPENAI_API_KEY=$(llm keys get openai) + """ + path = user_dir() / "keys.json" + if not path.exists(): + raise click.ClickException("No keys found") + keys = json.loads(path.read_text()) + try: + click.echo(keys[name]) + except KeyError: + raise click.ClickException("No key found with name '{}'".format(name)) + + +@keys.command(name="set") +@click.argument("name") +@click.option("--value", prompt="Enter key", hide_input=True, help="Value to set") +def keys_set(name, value): + """ + Save a key in the keys.json file + + Example usage: + + \b + $ llm keys set openai + Enter key: ... + """ + default = {"// Note": "This file stores secret API credentials. Do not share!"} + path = user_dir() / "keys.json" + path.parent.mkdir(parents=True, exist_ok=True) + if not path.exists(): + path.write_text(json.dumps(default)) + path.chmod(0o600) + try: + current = json.loads(path.read_text()) + except json.decoder.JSONDecodeError: + current = default + current[name] = value + path.write_text(json.dumps(current, indent=2) + "\n") + + +@cli.group( + cls=DefaultGroup, + default="list", + default_if_no_args=True, +) +def logs(): + "Tools for exploring logged prompts and responses" + + +@logs.command(name="path") +def logs_path(): + "Output the path to the logs.db file" + click.echo(logs_db_path()) + + +@logs.command(name="status") +def logs_status(): + "Show current status of database logging" + path = logs_db_path() + if not path.exists(): + click.echo("No log database found at {}".format(path)) + return + if logs_on(): + click.echo("Logging is ON for all prompts".format()) + else: + click.echo("Logging is OFF".format()) + db = sqlite_utils.Database(path) + migrate(db) + click.echo("Found log database at {}".format(path)) + click.echo("Number of conversations logged:\t{}".format(db["conversations"].count)) + click.echo("Number of responses logged:\t{}".format(db["responses"].count)) + click.echo( + "Database file size: \t\t{}".format(_human_readable_size(path.stat().st_size)) + ) + + +@logs.command(name="backup") +@click.argument("path", type=click.Path(dir_okay=True, writable=True)) +def backup(path): + "Backup your logs database to this file" + logs_path = logs_db_path() + path = pathlib.Path(path) + db = sqlite_utils.Database(logs_path) + try: + db.execute("vacuum into ?", [str(path)]) + except Exception as ex: + raise click.ClickException(str(ex)) + click.echo( + "Backed up {} to {}".format(_human_readable_size(path.stat().st_size), path) + ) + + +@logs.command(name="on") +def logs_turn_on(): + "Turn on logging for all prompts" + path = user_dir() / "logs-off" + if path.exists(): + path.unlink() + + +@logs.command(name="off") +def logs_turn_off(): + "Turn off logging for all prompts" + path = user_dir() / "logs-off" + path.touch() + + +LOGS_COLUMNS = """ responses.id, + responses.model, + responses.resolved_model, + responses.prompt, + responses.system, + responses.prompt_json, + responses.options_json, + responses.response, + responses.response_json, + responses.conversation_id, + responses.duration_ms, + responses.datetime_utc, + responses.input_tokens, + responses.output_tokens, + responses.token_details, + conversations.name as conversation_name, + conversations.model as conversation_model, + schemas.content as schema_json""" + +LOGS_SQL = """ +select +{columns} +from + responses +left join schemas on responses.schema_id = schemas.id +left join conversations on responses.conversation_id = conversations.id{extra_where} +order by {order_by}{limit} +""" +LOGS_SQL_SEARCH = """ +select +{columns} +from + responses +left join schemas on responses.schema_id = schemas.id +left join conversations on responses.conversation_id = conversations.id +join responses_fts on responses_fts.rowid = responses.rowid +where responses_fts match :query{extra_where} +order by {order_by}{limit} +""" + +ATTACHMENTS_SQL = """ +select + response_id, + attachments.id, + attachments.type, + attachments.path, + attachments.url, + length(attachments.content) as content_length +from attachments +join prompt_attachments + on attachments.id = prompt_attachments.attachment_id +where prompt_attachments.response_id in ({}) +order by prompt_attachments."order" +""" + + +@logs.command(name="list") +@click.option( + "-n", + "--count", + type=int, + default=None, + help="Number of entries to show - defaults to 3, use 0 for all", +) +@click.option( + "-p", + "--path", + type=click.Path(readable=True, exists=True, dir_okay=False), + help="Path to log database", + hidden=True, +) +@click.option( + "-d", + "--database", + type=click.Path(readable=True, exists=True, dir_okay=False), + help="Path to log database", +) +@click.option("-m", "--model", help="Filter by model or model alias") +@click.option("-q", "--query", help="Search for logs matching this string") +@click.option( + "fragments", + "--fragment", + "-f", + help="Filter for prompts using these fragments", + multiple=True, +) +@click.option( + "tools", + "-T", + "--tool", + multiple=True, + help="Filter for prompts with results from these tools", +) +@click.option( + "any_tools", + "--tools", + is_flag=True, + help="Filter for prompts with results from any tools", +) +@schema_option +@click.option( + "--schema-multi", + help="JSON schema used for multiple results", +) +@click.option( + "-l", "--latest", is_flag=True, help="Return latest results matching search query" +) +@click.option( + "--data", is_flag=True, help="Output newline-delimited JSON data for schema" +) +@click.option("--data-array", is_flag=True, help="Output JSON array of data for schema") +@click.option("--data-key", help="Return JSON objects from array in this key") +@click.option( + "--data-ids", is_flag=True, help="Attach corresponding IDs to JSON objects" +) +@click.option("-t", "--truncate", is_flag=True, help="Truncate long strings in output") +@click.option( + "-s", "--short", is_flag=True, help="Shorter YAML output with truncated prompts" +) +@click.option("-u", "--usage", is_flag=True, help="Include token usage") +@click.option("-r", "--response", is_flag=True, help="Just output the last response") +@click.option("-x", "--extract", is_flag=True, help="Extract first fenced code block") +@click.option( + "extract_last", + "--xl", + "--extract-last", + is_flag=True, + help="Extract last fenced code block", +) +@click.option( + "current_conversation", + "-c", + "--current", + is_flag=True, + flag_value=-1, + help="Show logs from the current conversation", +) +@click.option( + "conversation_id", + "--cid", + "--conversation", + help="Show logs for this conversation ID", +) +@click.option("--id-gt", help="Return responses with ID > this") +@click.option("--id-gte", help="Return responses with ID >= this") +@click.option( + "json_output", + "--json", + is_flag=True, + help="Output logs as JSON", +) +@click.option( + "--expand", + "-e", + is_flag=True, + help="Expand fragments to show their content", +) +def logs_list( + count, + path, + database, + model, + query, + fragments, + tools, + any_tools, + schema_input, + schema_multi, + latest, + data, + data_array, + data_key, + data_ids, + truncate, + short, + usage, + response, + extract, + extract_last, + current_conversation, + conversation_id, + id_gt, + id_gte, + json_output, + expand, +): + "Show logged prompts and their responses" + if database and not path: + path = database + path = pathlib.Path(path or logs_db_path()) + if not path.exists(): + raise click.ClickException("No log database found at {}".format(path)) + db = sqlite_utils.Database(path) + migrate(db) + + if schema_multi: + schema_input = schema_multi + schema = resolve_schema_input(db, schema_input, load_template) + if schema_multi: + schema = multi_schema(schema) + + if short and (json_output or response): + invalid = " or ".join( + [ + flag[0] + for flag in (("--json", json_output), ("--response", response)) + if flag[1] + ] + ) + raise click.ClickException("Cannot use --short and {} together".format(invalid)) + + if response and not current_conversation and not conversation_id: + current_conversation = True + + if current_conversation: + try: + conversation_id = next( + db.query( + "select conversation_id from responses order by id desc limit 1" + ) + )["conversation_id"] + except StopIteration: + # No conversations yet + raise click.ClickException("No conversations found") + + # For --conversation set limit 0, if not explicitly set + if count is None: + if conversation_id: + count = 0 + else: + count = 3 + + model_id = None + if model: + # Resolve alias, if any + try: + model_id = get_model(model).model_id + except UnknownModelError: + # Maybe they uninstalled a model, use the -m option as-is + model_id = model + + sql = LOGS_SQL + order_by = "responses.id desc" + if query: + sql = LOGS_SQL_SEARCH + if not latest: + order_by = "responses_fts.rank desc" + + limit = "" + if count is not None and count > 0: + limit = " limit {}".format(count) + + sql_format = { + "limit": limit, + "columns": LOGS_COLUMNS, + "extra_where": "", + "order_by": order_by, + } + where_bits = [] + sql_params = { + "model": model_id, + "query": query, + "conversation_id": conversation_id, + "id_gt": id_gt, + "id_gte": id_gte, + } + if model_id: + where_bits.append("responses.model = :model") + if conversation_id: + where_bits.append("responses.conversation_id = :conversation_id") + if id_gt: + where_bits.append("responses.id > :id_gt") + if id_gte: + where_bits.append("responses.id >= :id_gte") + if fragments: + # Resolve the fragments to their hashes + fragment_hashes = [ + fragment.id() for fragment in resolve_fragments(db, fragments) + ] + exists_clauses = [] + + for i, fragment_hash in enumerate(fragment_hashes): + exists_clause = f""" + exists ( + select 1 from prompt_fragments + where prompt_fragments.response_id = responses.id + and prompt_fragments.fragment_id in ( + select fragments.id from fragments + where hash = :f{i} + ) + union + select 1 from system_fragments + where system_fragments.response_id = responses.id + and system_fragments.fragment_id in ( + select fragments.id from fragments + where hash = :f{i} + ) + ) + """ + exists_clauses.append(exists_clause) + sql_params["f{}".format(i)] = fragment_hash + + where_bits.append(" and ".join(exists_clauses)) + + if any_tools: + # Any response that involved at least one tool result + where_bits.append(""" + exists ( + select 1 + from tool_results + where + tool_results.response_id = responses.id + ) + """) + if tools: + tools_by_name = get_tools() + # Filter responses by tools (must have ALL of the named tools, including plugin) + tool_clauses = [] + for i, tool_name in enumerate(tools): + try: + plugin_name = tools_by_name[tool_name].plugin + except KeyError: + raise click.ClickException(f"Unknown tool: {tool_name}") + + tool_clauses.append(f""" + exists ( + select 1 + from tool_results + join tools on tools.id = tool_results.tool_id + where tool_results.response_id = responses.id + and tools.name = :tool{i} + and tools.plugin = :plugin{i} + ) + """) + sql_params[f"tool{i}"] = tool_name + sql_params[f"plugin{i}"] = plugin_name + + # AND means “must have all” — use OR instead if you want “any of” + where_bits.append(" and ".join(tool_clauses)) + + schema_id = None + if schema: + schema_id = make_schema_id(schema)[0] + where_bits.append("responses.schema_id = :schema_id") + sql_params["schema_id"] = schema_id + + if where_bits: + where_ = " and " if query else " where " + sql_format["extra_where"] = where_ + " and ".join(where_bits) + + final_sql = sql.format(**sql_format) + rows = list(db.query(final_sql, sql_params)) + + # Reverse the order - we do this because we 'order by id desc limit 3' to get the + # 3 most recent results, but we still want to display them in chronological order + # ... except for searches where we don't do this + if not query and not data: + rows.reverse() + + # Fetch any attachments + ids = [row["id"] for row in rows] + attachments = list(db.query(ATTACHMENTS_SQL.format(",".join("?" * len(ids))), ids)) + attachments_by_id = {} + for attachment in attachments: + attachments_by_id.setdefault(attachment["response_id"], []).append(attachment) + + FRAGMENTS_SQL = """ + select + {table}.response_id, + fragments.hash, + fragments.id as fragment_id, + fragments.content, + ( + select json_group_array(fragment_aliases.alias) + from fragment_aliases + where fragment_aliases.fragment_id = fragments.id + ) as aliases + from {table} + join fragments on {table}.fragment_id = fragments.id + where {table}.response_id in ({placeholders}) + order by {table}."order" + """ + + # Fetch any prompt or system prompt fragments + prompt_fragments_by_id = {} + system_fragments_by_id = {} + for table, dictionary in ( + ("prompt_fragments", prompt_fragments_by_id), + ("system_fragments", system_fragments_by_id), + ): + for fragment in db.query( + FRAGMENTS_SQL.format(placeholders=",".join("?" * len(ids)), table=table), + ids, + ): + dictionary.setdefault(fragment["response_id"], []).append(fragment) + + if data or data_array or data_key or data_ids: + # Special case for --data to output valid JSON + to_output = [] + for row in rows: + response = row["response"] or "" + try: + decoded = json.loads(response) + new_items = [] + if ( + isinstance(decoded, dict) + and (data_key in decoded) + and all(isinstance(item, dict) for item in decoded[data_key]) + ): + for item in decoded[data_key]: + new_items.append(item) + else: + new_items.append(decoded) + if data_ids: + for item in new_items: + item[find_unused_key(item, "response_id")] = row["id"] + item[find_unused_key(item, "conversation_id")] = row["id"] + to_output.extend(new_items) + except ValueError: + pass + for line in output_rows_as_json(to_output, nl=not data_array, compact=True): + click.echo(line) + return + + # Tool usage information + TOOLS_SQL = """ + SELECT responses.id, + -- Tools related to this response + COALESCE( + (SELECT json_group_array(json_object( + 'id', t.id, + 'hash', t.hash, + 'name', t.name, + 'description', t.description, + 'input_schema', json(t.input_schema) + )) + FROM tools t + JOIN tool_responses tr ON t.id = tr.tool_id + WHERE tr.response_id = responses.id + ), + '[]' + ) AS tools, + -- Tool calls for this response + COALESCE( + (SELECT json_group_array(json_object( + 'id', tc.id, + 'tool_id', tc.tool_id, + 'name', tc.name, + 'arguments', json(tc.arguments), + 'tool_call_id', tc.tool_call_id + )) + FROM tool_calls tc + WHERE tc.response_id = responses.id + ), + '[]' + ) AS tool_calls, + -- Tool results for this response + COALESCE( + (SELECT json_group_array(json_object( + 'id', tr.id, + 'tool_id', tr.tool_id, + 'name', tr.name, + 'output', tr.output, + 'tool_call_id', tr.tool_call_id, + 'exception', tr.exception, + 'attachments', COALESCE( + (SELECT json_group_array(json_object( + 'id', a.id, + 'type', a.type, + 'path', a.path, + 'url', a.url, + 'content', a.content + )) + FROM tool_results_attachments tra + JOIN attachments a ON tra.attachment_id = a.id + WHERE tra.tool_result_id = tr.id + ), + '[]' + ) + )) + FROM tool_results tr + WHERE tr.response_id = responses.id + ), + '[]' + ) AS tool_results + FROM responses + where id in ({placeholders}) + """ + tool_info_by_id = { + row["id"]: { + "tools": json.loads(row["tools"]), + "tool_calls": json.loads(row["tool_calls"]), + "tool_results": json.loads(row["tool_results"]), + } + for row in db.query( + TOOLS_SQL.format(placeholders=",".join("?" * len(ids))), ids + ) + } + + for row in rows: + if truncate: + row["prompt"] = truncate_string(row["prompt"] or "") + row["response"] = truncate_string(row["response"] or "") + # Add prompt and system fragments + for key in ("prompt_fragments", "system_fragments"): + row[key] = [ + { + "hash": fragment["hash"], + "content": ( + fragment["content"] + if expand + else truncate_string(fragment["content"]) + ), + "aliases": json.loads(fragment["aliases"]), + } + for fragment in ( + prompt_fragments_by_id.get(row["id"], []) + if key == "prompt_fragments" + else system_fragments_by_id.get(row["id"], []) + ) + ] + # Either decode or remove all JSON keys + keys = list(row.keys()) + for key in keys: + if key.endswith("_json") and row[key] is not None: + if truncate: + del row[key] + else: + row[key] = json.loads(row[key]) + row.update(tool_info_by_id[row["id"]]) + + output = None + if json_output: + # Output as JSON if requested + for row in rows: + row["attachments"] = [ + {k: v for k, v in attachment.items() if k != "response_id"} + for attachment in attachments_by_id.get(row["id"], []) + ] + output = json.dumps(list(rows), indent=2) + elif extract or extract_last: + # Extract and return first code block + for row in rows: + output = extract_fenced_code_block(row["response"], last=extract_last) + if output is not None: + break + elif response: + # Just output the last response + if rows: + output = rows[-1]["response"] + + if output is not None: + click.echo(output) + else: + # Output neatly formatted human-readable logs + def _display_fragments(fragments, title): + if not fragments: + return + if not expand: + content = "\n".join( + ["- {}".format(fragment["hash"]) for fragment in fragments] + ) + else: + #
for each one + bits = [] + for fragment in fragments: + bits.append( + "
{}\n{}\n
".format( + fragment["hash"], maybe_fenced_code(fragment["content"]) + ) + ) + content = "\n".join(bits) + click.echo(f"\n### {title}\n\n{content}") + + current_system = None + should_show_conversation = True + for row in rows: + if short: + system = truncate_string( + row["system"] or "", 120, normalize_whitespace=True + ) + prompt = truncate_string( + row["prompt"] or "", 120, normalize_whitespace=True, keep_end=True + ) + cid = row["conversation_id"] + attachments = attachments_by_id.get(row["id"]) + obj = { + "model": row["model"], + "datetime": row["datetime_utc"].split(".")[0], + "conversation": cid, + } + if row["tool_calls"]: + obj["tool_calls"] = [ + "{}({})".format( + tool_call["name"], json.dumps(tool_call["arguments"]) + ) + for tool_call in row["tool_calls"] + ] + if row["tool_results"]: + obj["tool_results"] = [ + "{}: {}".format( + tool_result["name"], truncate_string(tool_result["output"]) + ) + for tool_result in row["tool_results"] + ] + if system: + obj["system"] = system + if prompt: + obj["prompt"] = prompt + if attachments: + items = [] + for attachment in attachments: + details = {"type": attachment["type"]} + if attachment.get("path"): + details["path"] = attachment["path"] + if attachment.get("url"): + details["url"] = attachment["url"] + items.append(details) + obj["attachments"] = items + for key in ("prompt_fragments", "system_fragments"): + obj[key] = [fragment["hash"] for fragment in row[key]] + if usage and (row["input_tokens"] or row["output_tokens"]): + usage_details = { + "input": row["input_tokens"], + "output": row["output_tokens"], + } + if row["token_details"]: + usage_details["details"] = json.loads(row["token_details"]) + obj["usage"] = usage_details + click.echo(yaml.dump([obj], sort_keys=False).strip()) + continue + # Not short, output Markdown + click.echo( + "# {}{}\n{}".format( + row["datetime_utc"].split(".")[0], + ( + " conversation: {} id: {}".format( + row["conversation_id"], row["id"] + ) + if should_show_conversation + else "" + ), + ( + ( + "\nModel: **{}**{}\n".format( + row["model"], + ( + " (resolved: **{}**)".format(row["resolved_model"]) + if row["resolved_model"] + else "" + ), + ) + ) + if should_show_conversation + else "" + ), + ) + ) + # In conversation log mode only show it for the first one + if conversation_id: + should_show_conversation = False + click.echo("## Prompt\n\n{}".format(row["prompt"] or "-- none --")) + _display_fragments(row["prompt_fragments"], "Prompt fragments") + if row["options_json"]: + options = row["options_json"] + if isinstance(options, str): + options = json.loads(options) + if options: + options_text = "\n".join( + "- {}: {}".format(key, value) for key, value in options.items() + ) + click.echo("\n## Options\n\n{}".format(options_text)) + if row["system"] != current_system: + if row["system"] is not None: + click.echo("\n## System\n\n{}".format(row["system"])) + current_system = row["system"] + _display_fragments(row["system_fragments"], "System fragments") + if row["schema_json"]: + click.echo( + "\n## Schema\n\n```json\n{}\n```".format( + json.dumps(row["schema_json"], indent=2) + ) + ) + # Show tool calls and results + if row["tools"]: + click.echo("\n### Tools\n") + for tool in row["tools"]: + click.echo( + "- **{}**: `{}`
\n {}
\n Arguments: {}".format( + tool["name"], + tool["hash"], + tool["description"], + json.dumps(tool["input_schema"]["properties"]), + ) + ) + if row["tool_results"]: + click.echo("\n### Tool results\n") + for tool_result in row["tool_results"]: + attachments = "" + for attachment in tool_result["attachments"]: + desc = "" + if attachment.get("type"): + desc += attachment["type"] + ": " + if attachment.get("path"): + desc += attachment["path"] + elif attachment.get("url"): + desc += attachment["url"] + elif attachment.get("content"): + desc += f"<{attachment['content_length']:,} bytes>" + attachments += "\n - {}".format(desc) + click.echo( + "- **{}**: `{}`
\n{}{}{}".format( + tool_result["name"], + tool_result["tool_call_id"], + textwrap.indent(tool_result["output"], " "), + ( + "
\n **Error**: {}\n".format( + tool_result["exception"] + ) + if tool_result["exception"] + else "" + ), + attachments, + ) + ) + attachments = attachments_by_id.get(row["id"]) + if attachments: + click.echo("\n### Attachments\n") + for i, attachment in enumerate(attachments, 1): + if attachment["path"]: + path = attachment["path"] + click.echo( + "{}. **{}**: `{}`".format(i, attachment["type"], path) + ) + elif attachment["url"]: + click.echo( + "{}. **{}**: {}".format( + i, attachment["type"], attachment["url"] + ) + ) + elif attachment["content_length"]: + click.echo( + "{}. **{}**: `<{} bytes>`".format( + i, + attachment["type"], + f"{attachment['content_length']:,}", + ) + ) + + # If a schema was provided and the row is valid JSON, pretty print and syntax highlight it + response = row["response"] + if row["schema_json"]: + try: + parsed = json.loads(response) + response = "```json\n{}\n```".format(json.dumps(parsed, indent=2)) + except ValueError: + pass + click.echo("\n## Response\n") + if row["tool_calls"]: + click.echo("### Tool calls\n") + for tool_call in row["tool_calls"]: + click.echo( + "- **{}**: `{}`
\n Arguments: {}".format( + tool_call["name"], + tool_call["tool_call_id"], + json.dumps(tool_call["arguments"]), + ) + ) + click.echo("") + if response: + click.echo("{}\n".format(response)) + if usage: + token_usage = token_usage_string( + row["input_tokens"], + row["output_tokens"], + json.loads(row["token_details"]) if row["token_details"] else None, + ) + if token_usage: + click.echo("## Token usage\n\n{}\n".format(token_usage)) + + +@cli.group( + cls=DefaultGroup, + default="list", + default_if_no_args=True, +) +def models(): + "Manage available models" + + +_type_lookup = { + "number": "float", + "integer": "int", + "string": "str", + "object": "dict", +} + + +@models.command(name="list") +@click.option( + "--options", is_flag=True, help="Show options for each model, if available" +) +@click.option("async_", "--async", is_flag=True, help="List async models") +@click.option("--schemas", is_flag=True, help="List models that support schemas") +@click.option("--tools", is_flag=True, help="List models that support tools") +@click.option( + "-q", + "--query", + multiple=True, + help="Search for models matching these strings", +) +@click.option("model_ids", "-m", "--model", help="Specific model IDs", multiple=True) +def models_list(options, async_, schemas, tools, query, model_ids): + "List available models" + models_that_have_shown_options = set() + for model_with_aliases in get_models_with_aliases(): + if async_ and not model_with_aliases.async_model: + continue + if query: + # Only show models where every provided query string matches + if not all(model_with_aliases.matches(q) for q in query): + continue + if model_ids: + ids_and_aliases = set( + [model_with_aliases.model.model_id] + model_with_aliases.aliases + ) + if not ids_and_aliases.intersection(model_ids): + continue + if schemas and not model_with_aliases.model.supports_schema: + continue + if tools and not model_with_aliases.model.supports_tools: + continue + extra_info = [] + if model_with_aliases.aliases: + extra_info.append( + "aliases: {}".format(", ".join(model_with_aliases.aliases)) + ) + model = ( + model_with_aliases.model if not async_ else model_with_aliases.async_model + ) + output = str(model) + if extra_info: + output += " ({})".format(", ".join(extra_info)) + if options and model.Options.model_json_schema()["properties"]: + output += "\n Options:" + for name, field in model.Options.model_json_schema()["properties"].items(): + any_of = field.get("anyOf") + if any_of is None: + any_of = [{"type": field.get("type", "str")}] + types = ", ".join( + [ + _type_lookup.get(item.get("type"), item.get("type", "str")) + for item in any_of + if item.get("type") != "null" + ] + ) + bits = ["\n ", name, ": ", types] + description = field.get("description", "") + if description and ( + model.__class__ not in models_that_have_shown_options + ): + wrapped = textwrap.wrap(description, 70) + bits.append("\n ") + bits.extend("\n ".join(wrapped)) + output += "".join(bits) + models_that_have_shown_options.add(model.__class__) + if options and model.attachment_types: + attachment_types = ", ".join(sorted(model.attachment_types)) + wrapper = textwrap.TextWrapper( + width=min(max(shutil.get_terminal_size().columns, 30), 70), + initial_indent=" ", + subsequent_indent=" ", + ) + output += "\n Attachment types:\n{}".format(wrapper.fill(attachment_types)) + features = ( + [] + + (["streaming"] if model.can_stream else []) + + (["schemas"] if model.supports_schema else []) + + (["tools"] if model.supports_tools else []) + + (["async"] if model_with_aliases.async_model else []) + ) + if options and features: + output += "\n Features:\n{}".format( + "\n".join(" - {}".format(feature) for feature in features) + ) + if options and hasattr(model, "needs_key") and model.needs_key: + output += "\n Keys:" + if hasattr(model, "needs_key") and model.needs_key: + output += "\n key: {}".format(model.needs_key) + if hasattr(model, "key_env_var") and model.key_env_var: + output += "\n env_var: {}".format(model.key_env_var) + click.echo(output) + if not query and not options and not schemas and not model_ids: + click.echo(f"Default: {get_default_model()}") + + +@models.command(name="default") +@click.argument("model", required=False) +def models_default(model): + "Show or set the default model" + if not model: + click.echo(get_default_model()) + return + # Validate it is a known model + try: + model = get_model(model) + set_default_model(model.model_id) + except KeyError: + raise click.ClickException("Unknown model: {}".format(model)) + + +@cli.group( + cls=DefaultGroup, + default="list", + default_if_no_args=True, +) +def templates(): + "Manage stored prompt templates" + + +@templates.command(name="list") +def templates_list(): + "List available prompt templates" + path = template_dir() + pairs = [] + for file in path.glob("*.yaml"): + name = file.stem + try: + template = load_template(name) + except LoadTemplateError: + # Skip invalid templates + continue + text = [] + if template.system: + text.append(f"system: {template.system}") + if template.prompt: + text.append(f" prompt: {template.prompt}") + else: + text = [template.prompt if template.prompt else ""] + pairs.append((name, "".join(text).replace("\n", " "))) + try: + max_name_len = max(len(p[0]) for p in pairs) + except ValueError: + return + else: + fmt = "{name:<" + str(max_name_len) + "} : {prompt}" + for name, prompt in sorted(pairs): + text = fmt.format(name=name, prompt=prompt) + click.echo(display_truncated(text)) + + +@templates.command(name="show") +@click.argument("name") +def templates_show(name): + "Show the specified prompt template" + try: + template = load_template(name) + except LoadTemplateError: + raise click.ClickException(f"Template '{name}' not found or invalid") + click.echo( + yaml.dump( + dict((k, v) for k, v in template.model_dump().items() if v is not None), + indent=4, + default_flow_style=False, + ) + ) + + +@templates.command(name="edit") +@click.argument("name") +def templates_edit(name): + "Edit the specified prompt template using the default $EDITOR" + # First ensure it exists + path = template_dir() / f"{name}.yaml" + if not path.exists(): + path.write_text(DEFAULT_TEMPLATE, "utf-8") + click.edit(filename=str(path)) + # Validate that template + load_template(name) + + +@templates.command(name="path") +def templates_path(): + "Output the path to the templates directory" + click.echo(template_dir()) + + +@templates.command(name="loaders") +def templates_loaders(): + "Show template loaders registered by plugins" + found = False + for prefix, loader in get_template_loaders().items(): + found = True + docs = "Undocumented" + if loader.__doc__: + docs = textwrap.dedent(loader.__doc__).strip() + click.echo(f"{prefix}:") + click.echo(textwrap.indent(docs, " ")) + if not found: + click.echo("No template loaders found") + + +@cli.group( + cls=DefaultGroup, + default="list", + default_if_no_args=True, +) +def schemas(): + "Manage stored schemas" + + +@schemas.command(name="list") +@click.option( + "-p", + "--path", + type=click.Path(readable=True, exists=True, dir_okay=False), + help="Path to log database", + hidden=True, +) +@click.option( + "-d", + "--database", + type=click.Path(readable=True, exists=True, dir_okay=False), + help="Path to log database", +) +@click.option( + "queries", + "-q", + "--query", + multiple=True, + help="Search for schemas matching this string", +) +@click.option("--full", is_flag=True, help="Output full schema contents") +@click.option("json_", "--json", is_flag=True, help="Output as JSON") +@click.option("nl", "--nl", is_flag=True, help="Output as newline-delimited JSON") +def schemas_list(path, database, queries, full, json_, nl): + "List stored schemas" + if database and not path: + path = database + path = pathlib.Path(path or logs_db_path()) + if not path.exists(): + raise click.ClickException("No log database found at {}".format(path)) + db = sqlite_utils.Database(path) + migrate(db) + + params = [] + where_sql = "" + if queries: + where_bits = ["schemas.content like ?" for _ in queries] + where_sql += " where {}".format(" and ".join(where_bits)) + params.extend("%{}%".format(q) for q in queries) + + sql = """ + select + schemas.id, + schemas.content, + max(responses.datetime_utc) as recently_used, + count(*) as times_used + from schemas + join responses + on responses.schema_id = schemas.id + {} group by responses.schema_id + order by recently_used + """.format(where_sql) + rows = db.query(sql, params) + + if json_ or nl: + for line in output_rows_as_json(rows, json_cols={"content"}, nl=nl): + click.echo(line) + return + + for row in rows: + click.echo("- id: {}".format(row["id"])) + if full: + click.echo( + " schema: |\n{}".format( + textwrap.indent( + json.dumps(json.loads(row["content"]), indent=2), " " + ) + ) + ) + else: + click.echo( + " summary: |\n {}".format( + schema_summary(json.loads(row["content"])) + ) + ) + click.echo( + " usage: |\n {} time{}, most recently {}".format( + row["times_used"], + "s" if row["times_used"] != 1 else "", + row["recently_used"], + ) + ) + + +@schemas.command(name="show") +@click.argument("schema_id") +@click.option( + "-p", + "--path", + type=click.Path(readable=True, exists=True, dir_okay=False), + help="Path to log database", + hidden=True, +) +@click.option( + "-d", + "--database", + type=click.Path(readable=True, exists=True, dir_okay=False), + help="Path to log database", +) +def schemas_show(schema_id, path, database): + "Show a stored schema" + if database and not path: + path = database + path = pathlib.Path(path or logs_db_path()) + if not path.exists(): + raise click.ClickException("No log database found at {}".format(path)) + db = sqlite_utils.Database(path) + migrate(db) + + try: + row = db["schemas"].get(schema_id) + except sqlite_utils.db.NotFoundError: + raise click.ClickException("Invalid schema ID") + click.echo(json.dumps(json.loads(row["content"]), indent=2)) + + +@schemas.command(name="dsl") +@click.argument("input") +@click.option("--multi", is_flag=True, help="Wrap in an array") +def schemas_dsl_debug(input, multi): + """ + Convert LLM's schema DSL to a JSON schema + + \b + llm schema dsl 'name, age int, bio: their bio' + """ + schema = schema_dsl(input, multi) + click.echo(json.dumps(schema, indent=2)) + + +@cli.group( + cls=DefaultGroup, + default="list", + default_if_no_args=True, +) +def tools(): + "Manage tools that can be made available to LLMs" + + +@tools.command(name="list") +@click.argument("tool_defs", nargs=-1) +@click.option("json_", "--json", is_flag=True, help="Output as JSON") +@click.option( + "python_tools", + "--functions", + help="Python code block or file path defining functions to register as tools", + multiple=True, +) +def tools_list(tool_defs, json_, python_tools): + "List available tools that have been provided by plugins" + + def introspect_tools(toolbox_class): + methods = [] + for tool in toolbox_class.method_tools(): + methods.append( + { + "name": tool.name, + "description": tool.description, + "arguments": tool.input_schema, + "implementation": tool.implementation, + } + ) + return methods + + if tool_defs: + tools = {} + for tool in _gather_tools(tool_defs, python_tools): + if hasattr(tool, "name"): + tools[tool.name] = tool + else: + tools[tool.__class__.__name__] = tool + else: + tools = get_tools() + if python_tools: + for code_or_path in python_tools: + for tool in _tools_from_code(code_or_path): + tools[tool.name] = tool + + output_tools = [] + output_toolboxes = [] + tool_objects = [] + toolbox_objects = [] + for name, tool in sorted(tools.items()): + if isinstance(tool, Tool): + tool_objects.append(tool) + output_tools.append( + { + "name": name, + "description": tool.description, + "arguments": tool.input_schema, + "plugin": tool.plugin, + } + ) + else: + toolbox_objects.append(tool) + output_toolboxes.append( + { + "name": name, + "tools": [ + { + "name": tool["name"], + "description": tool["description"], + "arguments": tool["arguments"], + } + for tool in introspect_tools(tool) + ], + } + ) + if json_: + click.echo( + json.dumps( + {"tools": output_tools, "toolboxes": output_toolboxes}, + indent=2, + ) + ) + else: + for tool in tool_objects: + sig = "()" + if tool.implementation: + sig = str(inspect.signature(tool.implementation)) + click.echo( + "{}{}{}\n".format( + tool.name, + sig, + " (plugin: {})".format(tool.plugin) if tool.plugin else "", + ) + ) + if tool.description: + click.echo(textwrap.indent(tool.description.strip(), " ") + "\n") + for toolbox in toolbox_objects: + click.echo(toolbox.name + ":\n") + for tool in toolbox.method_tools(): + sig = ( + str(inspect.signature(tool.implementation)) + .replace("(self, ", "(") + .replace("(self)", "()") + ) + click.echo( + " {}{}\n".format( + tool.name, + sig, + ) + ) + if tool.description: + click.echo(textwrap.indent(tool.description.strip(), " ") + "\n") + + +@cli.group( + cls=DefaultGroup, + default="list", + default_if_no_args=True, +) +def aliases(): + "Manage model aliases" + + +@aliases.command(name="list") +@click.option("json_", "--json", is_flag=True, help="Output as JSON") +def aliases_list(json_): + "List current aliases" + to_output = [] + for alias, model in get_model_aliases().items(): + if alias != model.model_id: + to_output.append((alias, model.model_id, "")) + for alias, embedding_model in get_embedding_model_aliases().items(): + if alias != embedding_model.model_id: + to_output.append((alias, embedding_model.model_id, "embedding")) + if json_: + click.echo( + json.dumps({key: value for key, value, type_ in to_output}, indent=4) + ) + return + max_alias_length = max(len(a) for a, _, _ in to_output) + fmt = "{alias:<" + str(max_alias_length) + "} : {model_id}{type_}" + for alias, model_id, type_ in to_output: + click.echo( + fmt.format( + alias=alias, model_id=model_id, type_=f" ({type_})" if type_ else "" + ) + ) + + +@aliases.command(name="set") +@click.argument("alias") +@click.argument("model_id", required=False) +@click.option( + "-q", + "--query", + multiple=True, + help="Set alias for model matching these strings", +) +def aliases_set(alias, model_id, query): + """ + Set an alias for a model + + Example usage: + + \b + llm aliases set mini gpt-4o-mini + + Alternatively you can omit the model ID and specify one or more -q options. + The first model matching all of those query strings will be used. + + \b + llm aliases set mini -q 4o -q mini + """ + if not model_id: + if not query: + raise click.ClickException( + "You must provide a model_id or at least one -q option" + ) + # Search for the first model matching all query strings + found = None + for model_with_aliases in get_models_with_aliases(): + if all(model_with_aliases.matches(q) for q in query): + found = model_with_aliases + break + if not found: + raise click.ClickException( + "No model found matching query: " + ", ".join(query) + ) + model_id = found.model.model_id + set_alias(alias, model_id) + click.echo( + f"Alias '{alias}' set to model '{model_id}'", + err=True, + ) + else: + set_alias(alias, model_id) + + +@aliases.command(name="remove") +@click.argument("alias") +def aliases_remove(alias): + """ + Remove an alias + + Example usage: + + \b + $ llm aliases remove turbo + """ + try: + remove_alias(alias) + except KeyError as ex: + raise click.ClickException(ex.args[0]) + + +@aliases.command(name="path") +def aliases_path(): + "Output the path to the aliases.json file" + click.echo(user_dir() / "aliases.json") + + +@cli.group( + cls=DefaultGroup, + default="list", + default_if_no_args=True, +) +def fragments(): + """ + Manage fragments that are stored in the database + + Fragments are reusable snippets of text that are shared across multiple prompts. + """ + + +@fragments.command(name="list") +@click.option( + "queries", + "-q", + "--query", + multiple=True, + help="Search for fragments matching these strings", +) +@click.option("--aliases", is_flag=True, help="Show only fragments with aliases") +@click.option("json_", "--json", is_flag=True, help="Output as JSON") +def fragments_list(queries, aliases, json_): + "List current fragments" + db = sqlite_utils.Database(logs_db_path()) + migrate(db) + params = {} + param_count = 0 + where_bits = [] + if aliases: + where_bits.append("fragment_aliases.alias is not null") + for q in queries: + param_count += 1 + p = f"p{param_count}" + params[p] = q + where_bits.append(f""" + (fragments.hash = :{p} or fragment_aliases.alias = :{p} + or fragments.source like '%' || :{p} || '%' + or fragments.content like '%' || :{p} || '%') + """) + where = "\n and\n ".join(where_bits) + if where: + where = " where " + where + sql = """ + select + fragments.hash, + json_group_array(fragment_aliases.alias) filter ( + where + fragment_aliases.alias is not null + ) as aliases, + fragments.datetime_utc, + fragments.source, + fragments.content + from + fragments + left join + fragment_aliases on fragment_aliases.fragment_id = fragments.id + {where} + group by + fragments.id, fragments.hash, fragments.content, fragments.datetime_utc, fragments.source + order by fragments.datetime_utc + """.format(where=where) + results = list(db.query(sql, params)) + for result in results: + result["aliases"] = json.loads(result["aliases"]) + if json_: + click.echo(json.dumps(results, indent=4)) + else: + yaml.add_representer( + str, + lambda dumper, data: dumper.represent_scalar( + "tag:yaml.org,2002:str", data, style="|" if "\n" in data else None + ), + ) + for result in results: + result["content"] = truncate_string(result["content"]) + click.echo(yaml.dump([result], sort_keys=False, width=sys.maxsize).strip()) + + +@fragments.command(name="set") +@click.argument("alias", callback=validate_fragment_alias) +@click.argument("fragment") +def fragments_set(alias, fragment): + """ + Set an alias for a fragment + + Accepts an alias and a file path, URL, hash or '-' for stdin + + Example usage: + + \b + llm fragments set mydocs ./docs.md + """ + db = sqlite_utils.Database(logs_db_path()) + migrate(db) + try: + resolved = resolve_fragments(db, [fragment])[0] + except FragmentNotFound as ex: + raise click.ClickException(str(ex)) + migrate(db) + alias_sql = """ + insert into fragment_aliases (alias, fragment_id) + values (:alias, :fragment_id) + on conflict(alias) do update set + fragment_id = excluded.fragment_id; + """ + with db.conn: + fragment_id = ensure_fragment(db, resolved) + db.conn.execute(alias_sql, {"alias": alias, "fragment_id": fragment_id}) + + +@fragments.command(name="show") +@click.argument("alias_or_hash") +def fragments_show(alias_or_hash): + """ + Display the fragment stored under an alias or hash + + \b + llm fragments show mydocs + """ + db = sqlite_utils.Database(logs_db_path()) + migrate(db) + try: + resolved = resolve_fragments(db, [alias_or_hash])[0] + except FragmentNotFound as ex: + raise click.ClickException(str(ex)) + click.echo(resolved) + + +@fragments.command(name="remove") +@click.argument("alias", callback=validate_fragment_alias) +def fragments_remove(alias): + """ + Remove a fragment alias + + Example usage: + + \b + llm fragments remove docs + """ + db = sqlite_utils.Database(logs_db_path()) + migrate(db) + with db.conn: + db.conn.execute( + "delete from fragment_aliases where alias = :alias", {"alias": alias} + ) + + +@fragments.command(name="loaders") +def fragments_loaders(): + """Show fragment loaders registered by plugins""" + from llm import get_fragment_loaders + + found = False + for prefix, loader in get_fragment_loaders().items(): + if found: + # Extra newline on all after the first + click.echo("") + found = True + docs = "Undocumented" + if loader.__doc__: + docs = textwrap.dedent(loader.__doc__).strip() + click.echo(f"{prefix}:") + click.echo(textwrap.indent(docs, " ")) + if not found: + click.echo("No fragment loaders found") + + +@cli.command(name="plugins") +@click.option("--all", help="Include built-in default plugins", is_flag=True) +@click.option( + "hooks", "--hook", help="Filter for plugins that implement this hook", multiple=True +) +def plugins_list(all, hooks): + "List installed plugins" + plugins = get_plugins(all) + hooks = set(hooks) + if hooks: + plugins = [plugin for plugin in plugins if hooks.intersection(plugin["hooks"])] + click.echo(json.dumps(plugins, indent=2)) + + +def display_truncated(text): + console_width = shutil.get_terminal_size()[0] + if len(text) > console_width: + return text[: console_width - 3] + "..." + else: + return text + + +@cli.command() +@click.argument("packages", nargs=-1, required=False) +@click.option( + "-U", "--upgrade", is_flag=True, help="Upgrade packages to latest version" +) +@click.option( + "-e", + "--editable", + help="Install a project in editable mode from this path", +) +@click.option( + "--force-reinstall", + is_flag=True, + help="Reinstall all packages even if they are already up-to-date", +) +@click.option( + "--no-cache-dir", + is_flag=True, + help="Disable the cache", +) +@click.option( + "--pre", + is_flag=True, + help="Include pre-release and development versions", +) +def install(packages, upgrade, editable, force_reinstall, no_cache_dir, pre): + """Install packages from PyPI into the same environment as LLM""" + args = ["pip", "install"] + if upgrade: + args += ["--upgrade"] + if editable: + args += ["--editable", editable] + if force_reinstall: + args += ["--force-reinstall"] + if no_cache_dir: + args += ["--no-cache-dir"] + if pre: + args += ["--pre"] + args += list(packages) + sys.argv = args + run_module("pip", run_name="__main__") + + +@cli.command() +@click.argument("packages", nargs=-1, required=True) +@click.option("-y", "--yes", is_flag=True, help="Don't ask for confirmation") +def uninstall(packages, yes): + """Uninstall Python packages from the LLM environment""" + sys.argv = ["pip", "uninstall"] + list(packages) + (["-y"] if yes else []) + run_module("pip", run_name="__main__") + + +@cli.command() +@click.argument("collection", required=False) +@click.argument("id", required=False) +@click.option( + "-i", + "--input", + type=click.Path(exists=True, readable=True, allow_dash=True), + help="File to embed", +) +@click.option( + "-m", "--model", help="Embedding model to use", envvar="LLM_EMBEDDING_MODEL" +) +@click.option("--store", is_flag=True, help="Store the text itself in the database") +@click.option( + "-d", + "--database", + type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), + envvar="LLM_EMBEDDINGS_DB", +) +@click.option( + "-c", + "--content", + help="Content to embed", +) +@click.option("--binary", is_flag=True, help="Treat input as binary data") +@click.option( + "--metadata", + help="JSON object metadata to store", + callback=json_validator("metadata"), +) +@click.option( + "format_", + "-f", + "--format", + type=click.Choice(["json", "blob", "base64", "hex"]), + help="Output format", +) +def embed( + collection, id, input, model, store, database, content, binary, metadata, format_ +): + """Embed text and store or return the result""" + if collection and not id: + raise click.ClickException("Must provide both collection and id") + + if store and not collection: + raise click.ClickException("Must provide collection when using --store") + + # Lazy load this because we do not need it for -c or -i versions + def get_db(): + if database: + return sqlite_utils.Database(database) + else: + return sqlite_utils.Database(user_dir() / "embeddings.db") + + collection_obj = None + model_obj = None + if collection: + db = get_db() + if Collection.exists(db, collection): + # Load existing collection and use its model + collection_obj = Collection(collection, db) + model_obj = collection_obj.model() + else: + # We will create a new one, but that means model is required + if not model: + model = get_default_embedding_model() + if model is None: + raise click.ClickException( + "You need to specify an embedding model (no default model is set)" + ) + collection_obj = Collection(collection, db=db, model_id=model) + model_obj = collection_obj.model() + + if model_obj is None: + if model is None: + model = get_default_embedding_model() + try: + model_obj = get_embedding_model(model) + except UnknownModelError: + raise click.ClickException( + "You need to specify an embedding model (no default model is set)" + ) + + show_output = True + if collection and (format_ is None): + show_output = False + + # Resolve input text + if not content: + if not input or input == "-": + # Read from stdin + input_source = sys.stdin.buffer if binary else sys.stdin + content = input_source.read() + else: + mode = "rb" if binary else "r" + with open(input, mode) as f: + content = f.read() + + if not content: + raise click.ClickException("No content provided") + + if collection_obj: + embedding = collection_obj.embed(id, content, metadata=metadata, store=store) + else: + embedding = model_obj.embed(content) + + if show_output: + if format_ == "json" or format_ is None: + click.echo(json.dumps(embedding)) + elif format_ == "blob": + click.echo(encode(embedding)) + elif format_ == "base64": + click.echo(base64.b64encode(encode(embedding)).decode("ascii")) + elif format_ == "hex": + click.echo(encode(embedding).hex()) + + +@cli.command() +@click.argument("collection") +@click.argument( + "input_path", + type=click.Path(exists=True, dir_okay=False, allow_dash=True, readable=True), + required=False, +) +@click.option( + "--format", + type=click.Choice(["json", "csv", "tsv", "nl"]), + help="Format of input file - defaults to auto-detect", +) +@click.option( + "--files", + type=(click.Path(file_okay=False, dir_okay=True, allow_dash=False), str), + multiple=True, + help="Embed files in this directory - specify directory and glob pattern", +) +@click.option( + "encodings", + "--encoding", + help="Encodings to try when reading --files", + multiple=True, +) +@click.option("--binary", is_flag=True, help="Treat --files as binary data") +@click.option("--sql", help="Read input using this SQL query") +@click.option( + "--attach", + type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)), + multiple=True, + help="Additional databases to attach - specify alias and file path", +) +@click.option( + "--batch-size", type=int, help="Batch size to use when running embeddings" +) +@click.option("--prefix", help="Prefix to add to the IDs", default="") +@click.option( + "-m", "--model", help="Embedding model to use", envvar="LLM_EMBEDDING_MODEL" +) +@click.option( + "--prepend", + help="Prepend this string to all content before embedding", +) +@click.option("--store", is_flag=True, help="Store the text itself in the database") +@click.option( + "-d", + "--database", + type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), + envvar="LLM_EMBEDDINGS_DB", +) +def embed_multi( + collection, + input_path, + format, + files, + encodings, + binary, + sql, + attach, + batch_size, + prefix, + model, + prepend, + store, + database, +): + """ + Store embeddings for multiple strings at once in the specified collection. + + Input data can come from one of three sources: + + \b + 1. A CSV, TSV, JSON or JSONL file: + - CSV/TSV: First column is ID, remaining columns concatenated as content + - JSON: Array of objects with "id" field and content fields + - JSONL: Newline-delimited JSON objects + + \b + Examples: + llm embed-multi docs input.csv + cat data.json | llm embed-multi docs - + llm embed-multi docs input.json --format json + + \b + 2. A SQL query against a SQLite database: + - First column returned is used as ID + - Other columns concatenated to form content + + \b + Examples: + llm embed-multi docs --sql "SELECT id, title, body FROM posts" + llm embed-multi docs --attach blog blog.db --sql "SELECT id, content FROM blog.posts" + + \b + 3. Files in directories matching glob patterns: + - Each file becomes one embedding + - Relative file paths become IDs + + \b + Examples: + llm embed-multi docs --files docs '**/*.md' + llm embed-multi images --files photos '*.jpg' --binary + llm embed-multi texts --files texts '*.txt' --encoding utf-8 --encoding latin-1 + """ + if binary and not files: + raise click.UsageError("--binary must be used with --files") + if binary and encodings: + raise click.UsageError("--binary cannot be used with --encoding") + if not input_path and not sql and not files: + raise click.UsageError("Either --sql or input path or --files is required") + + if files: + if input_path or sql or format: + raise click.UsageError( + "Cannot use --files with --sql, input path or --format" + ) + + if database: + db = sqlite_utils.Database(database) + else: + db = sqlite_utils.Database(user_dir() / "embeddings.db") + + for alias, attach_path in attach: + db.attach(alias, attach_path) + + try: + collection_obj = Collection( + collection, db=db, model_id=model or get_default_embedding_model() + ) + except ValueError: + raise click.ClickException( + "You need to specify an embedding model (no default model is set)" + ) + + expected_length = None + if files: + encodings = encodings or ("utf-8", "latin-1") + + def count_files(): + i = 0 + for directory, pattern in files: + for path in pathlib.Path(directory).glob(pattern): + i += 1 + return i + + def iterate_files(): + for directory, pattern in files: + p = pathlib.Path(directory) + if not p.exists() or not p.is_dir(): + # fixes issue/274 - raise error if directory does not exist + raise click.UsageError(f"Invalid directory: {directory}") + for path in pathlib.Path(directory).glob(pattern): + if path.is_dir(): + continue # fixed issue/280 - skip directories + relative = path.relative_to(directory) + content = None + if binary: + content = path.read_bytes() + else: + for encoding in encodings: + try: + content = path.read_text(encoding=encoding) + except UnicodeDecodeError: + continue + if content is None: + # Log to stderr + click.echo( + "Could not decode text in file {}".format(path), + err=True, + ) + else: + yield {"id": str(relative), "content": content} + + expected_length = count_files() + rows = iterate_files() + elif sql: + rows = db.query(sql) + count_sql = "select count(*) as c from ({})".format(sql) + expected_length = next(db.query(count_sql))["c"] + else: + + def load_rows(fp): + return rows_from_file(fp, Format[format.upper()] if format else None)[0] + + try: + if input_path != "-": + # Read the file twice - first time is to get a count + expected_length = 0 + with open(input_path, "rb") as fp: + for _ in load_rows(fp): + expected_length += 1 + + rows = load_rows( + open(input_path, "rb") + if input_path != "-" + else io.BufferedReader(sys.stdin.buffer) + ) + except json.JSONDecodeError as ex: + raise click.ClickException(str(ex)) + + with click.progressbar( + rows, label="Embedding", show_percent=True, length=expected_length + ) as rows: + + def tuples() -> Iterable[Tuple[str, Union[bytes, str]]]: + for row in rows: + values = list(row.values()) + id: str = prefix + str(values[0]) + content: Optional[Union[bytes, str]] = None + if binary: + content = cast(bytes, values[1]) + else: + content = " ".join(v or "" for v in values[1:]) + if prepend and isinstance(content, str): + content = prepend + content + yield id, content or "" + + embed_kwargs = {"store": store} + if batch_size: + embed_kwargs["batch_size"] = batch_size + collection_obj.embed_multi(tuples(), **embed_kwargs) + + +@cli.command() +@click.argument("collection") +@click.argument("id", required=False) +@click.option( + "-i", + "--input", + type=click.Path(exists=True, readable=True, allow_dash=True), + help="File to embed for comparison", +) +@click.option("-c", "--content", help="Content to embed for comparison") +@click.option("--binary", is_flag=True, help="Treat input as binary data") +@click.option( + "-n", "--number", type=int, default=10, help="Number of results to return" +) +@click.option("-p", "--plain", is_flag=True, help="Output in plain text format") +@click.option( + "-d", + "--database", + type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), + envvar="LLM_EMBEDDINGS_DB", +) +@click.option("--prefix", help="Just IDs with this prefix", default="") +def similar(collection, id, input, content, binary, number, plain, database, prefix): + """ + Return top N similar IDs from a collection using cosine similarity. + + Example usage: + + \b + llm similar my-collection -c "I like cats" + + Or to find content similar to a specific stored ID: + + \b + llm similar my-collection 1234 + """ + if not id and not content and not input: + raise click.ClickException("Must provide content or an ID for the comparison") + + if database: + db = sqlite_utils.Database(database) + else: + db = sqlite_utils.Database(user_dir() / "embeddings.db") + + if not db["embeddings"].exists(): + raise click.ClickException("No embeddings table found in database") + + try: + collection_obj = Collection(collection, db, create=False) + except Collection.DoesNotExist: + raise click.ClickException("Collection does not exist") + + if id: + try: + results = collection_obj.similar_by_id(id, number, prefix=prefix) + except Collection.DoesNotExist: + raise click.ClickException("ID not found in collection") + else: + # Resolve input text + if not content: + if not input or input == "-": + # Read from stdin + input_source = sys.stdin.buffer if binary else sys.stdin + content = input_source.read() + else: + mode = "rb" if binary else "r" + with open(input, mode) as f: + content = f.read() + if not content: + raise click.ClickException("No content provided") + results = collection_obj.similar(content, number, prefix=prefix) + + for result in results: + if plain: + click.echo(f"{result.id} ({result.score})\n") + if result.content: + click.echo(textwrap.indent(result.content, " ")) + if result.metadata: + click.echo(textwrap.indent(json.dumps(result.metadata), " ")) + click.echo("") + else: + click.echo(json.dumps(asdict(result))) + + +@cli.group( + cls=DefaultGroup, + default="list", + default_if_no_args=True, +) +def embed_models(): + "Manage available embedding models" + + +@embed_models.command(name="list") +@click.option( + "-q", + "--query", + multiple=True, + help="Search for embedding models matching these strings", +) +def embed_models_list(query): + "List available embedding models" + output = [] + for model_with_aliases in get_embedding_models_with_aliases(): + if query: + if not all(model_with_aliases.matches(q) for q in query): + continue + s = str(model_with_aliases.model) + if model_with_aliases.aliases: + s += " (aliases: {})".format(", ".join(model_with_aliases.aliases)) + output.append(s) + click.echo("\n".join(output)) + + +@embed_models.command(name="default") +@click.argument("model", required=False) +@click.option( + "--remove-default", is_flag=True, help="Reset to specifying no default model" +) +def embed_models_default(model, remove_default): + "Show or set the default embedding model" + if not model and not remove_default: + default = get_default_embedding_model() + if default is None: + click.echo("", err=True) + else: + click.echo(default) + return + # Validate it is a known model + try: + if remove_default: + set_default_embedding_model(None) + else: + model = get_embedding_model(model) + set_default_embedding_model(model.model_id) + except KeyError: + raise click.ClickException("Unknown embedding model: {}".format(model)) + + +@cli.group( + cls=DefaultGroup, + default="list", + default_if_no_args=True, +) +def collections(): + "View and manage collections of embeddings" + + +@collections.command(name="path") +def collections_path(): + "Output the path to the embeddings database" + click.echo(user_dir() / "embeddings.db") + + +@collections.command(name="list") +@click.option( + "-d", + "--database", + type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), + envvar="LLM_EMBEDDINGS_DB", + help="Path to embeddings database", +) +@click.option("json_", "--json", is_flag=True, help="Output as JSON") +def embed_db_collections(database, json_): + "View a list of collections" + database = database or (user_dir() / "embeddings.db") + db = sqlite_utils.Database(str(database)) + if not db["collections"].exists(): + raise click.ClickException("No collections table found in {}".format(database)) + rows = db.query(""" + select + collections.name, + collections.model, + count(embeddings.id) as num_embeddings + from + collections left join embeddings + on collections.id = embeddings.collection_id + group by + collections.name, collections.model + """) + if json_: + click.echo(json.dumps(list(rows), indent=4)) + else: + for row in rows: + click.echo("{}: {}".format(row["name"], row["model"])) + click.echo( + " {} embedding{}".format( + row["num_embeddings"], "s" if row["num_embeddings"] != 1 else "" + ) + ) + + +@collections.command(name="delete") +@click.argument("collection") +@click.option( + "-d", + "--database", + type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), + envvar="LLM_EMBEDDINGS_DB", + help="Path to embeddings database", +) +def collections_delete(collection, database): + """ + Delete the specified collection + + Example usage: + + \b + llm collections delete my-collection + """ + database = database or (user_dir() / "embeddings.db") + db = sqlite_utils.Database(str(database)) + try: + collection_obj = Collection(collection, db, create=False) + except Collection.DoesNotExist: + raise click.ClickException("Collection does not exist") + collection_obj.delete() + + +@models.group( + cls=DefaultGroup, + default="list", + default_if_no_args=True, +) +def options(): + "Manage default options for models" + + +@options.command(name="list") +def options_list(): + """ + List default options for all models + + Example usage: + + \b + llm models options list + """ + options = get_all_model_options() + if not options: + click.echo("No default options set for any models.", err=True) + return + + for model_id, model_options in options.items(): + click.echo(f"{model_id}:") + for key, value in model_options.items(): + click.echo(f" {key}: {value}") + + +@options.command(name="show") +@click.argument("model") +def options_show(model): + """ + List default options set for a specific model + + Example usage: + + \b + llm models options show gpt-4o + """ + import llm + + try: + # Resolve alias to model ID + model_obj = llm.get_model(model) + model_id = model_obj.model_id + except llm.UnknownModelError: + # Use as-is if not found + model_id = model + + options = get_model_options(model_id) + if not options: + click.echo(f"No default options set for model '{model_id}'.", err=True) + return + + for key, value in options.items(): + click.echo(f"{key}: {value}") + + +@options.command(name="set") +@click.argument("model") +@click.argument("key") +@click.argument("value") +def options_set(model, key, value): + """ + Set a default option for a model + + Example usage: + + \b + llm models options set gpt-4o temperature 0.5 + """ + import llm + + try: + # Resolve alias to model ID + model_obj = llm.get_model(model) + model_id = model_obj.model_id + + # Validate option against model schema + try: + # Create a test Options object to validate + test_options = {key: value} + model_obj.Options(**test_options) + except pydantic.ValidationError as ex: + raise click.ClickException(render_errors(ex.errors())) + + except llm.UnknownModelError: + # Use as-is if not found + model_id = model + + set_model_option(model_id, key, value) + click.echo(f"Set default option {key}={value} for model {model_id}", err=True) + + +@options.command(name="clear") +@click.argument("model") +@click.argument("key", required=False) +def options_clear(model, key): + """ + Clear default option(s) for a model + + Example usage: + + \b + llm models options clear gpt-4o + # Or for a single option + llm models options clear gpt-4o temperature + """ + import llm + + try: + # Resolve alias to model ID + model_obj = llm.get_model(model) + model_id = model_obj.model_id + except llm.UnknownModelError: + # Use as-is if not found + model_id = model + + cleared_keys = [] + if not key: + cleared_keys = list(get_model_options(model_id).keys()) + for key_ in cleared_keys: + clear_model_option(model_id, key_) + else: + cleared_keys.append(key) + clear_model_option(model_id, key) + if cleared_keys: + if len(cleared_keys) == 1: + click.echo(f"Cleared option '{cleared_keys[0]}' for model {model_id}") + else: + click.echo( + f"Cleared {', '.join(cleared_keys)} options for model {model_id}" + ) + + +def template_dir(): + path = user_dir() / "templates" + path.mkdir(parents=True, exist_ok=True) + return path + + +def logs_db_path(): + return user_dir() / "logs.db" + + +def get_history(chat_id): + if chat_id is None: + return None, [] + log_path = logs_db_path() + db = sqlite_utils.Database(log_path) + migrate(db) + if chat_id == -1: + # Return the most recent chat + last_row = list(db["logs"].rows_where(order_by="-id", limit=1)) + if last_row: + chat_id = last_row[0].get("chat_id") or last_row[0].get("id") + else: # Database is empty + return None, [] + rows = db["logs"].rows_where( + "id = ? or chat_id = ?", [chat_id, chat_id], order_by="id" + ) + return chat_id, rows + + +def render_errors(errors): + output = [] + for error in errors: + output.append(", ".join(error["loc"])) + output.append(" " + error["msg"]) + return "\n".join(output) + + +load_plugins() + +pm.hook.register_commands(cli=cli) + + +def _human_readable_size(size_bytes): + if size_bytes == 0: + return "0B" + + size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") + i = 0 + + while size_bytes >= 1024 and i < len(size_name) - 1: + size_bytes /= 1024.0 + i += 1 + + return "{:.2f}{}".format(size_bytes, size_name[i]) + + +def logs_on(): + return not (user_dir() / "logs-off").exists() + + +def get_all_model_options() -> dict: + """ + Get all default options for all models + """ + path = user_dir() / "model_options.json" + if not path.exists(): + return {} + + try: + options = json.loads(path.read_text()) + except json.JSONDecodeError: + return {} + + return options + + +def get_model_options(model_id: str) -> dict: + """ + Get default options for a specific model + + Args: + model_id: Return options for model with this ID + + Returns: + A dictionary of model options + """ + path = user_dir() / "model_options.json" + if not path.exists(): + return {} + + try: + options = json.loads(path.read_text()) + except json.JSONDecodeError: + return {} + + return options.get(model_id, {}) + + +def set_model_option(model_id: str, key: str, value: Any) -> None: + """ + Set a default option for a model. + + Args: + model_id: The model ID + key: The option key + value: The option value + """ + path = user_dir() / "model_options.json" + if path.exists(): + try: + options = json.loads(path.read_text()) + except json.JSONDecodeError: + options = {} + else: + options = {} + + # Ensure the model has an entry + if model_id not in options: + options[model_id] = {} + + # Set the option + options[model_id][key] = value + + # Save the options + path.write_text(json.dumps(options, indent=2)) + + +def clear_model_option(model_id: str, key: str) -> None: + """ + Clear a model option + + Args: + model_id: The model ID + key: Key to clear + """ + path = user_dir() / "model_options.json" + if not path.exists(): + return + + try: + options = json.loads(path.read_text()) + except json.JSONDecodeError: + return + + if model_id not in options: + return + + if key in options[model_id]: + del options[model_id][key] + if not options[model_id]: + del options[model_id] + + path.write_text(json.dumps(options, indent=2)) + + +class LoadTemplateError(ValueError): + pass + + +def _parse_yaml_template(name, content): + try: + loaded = yaml.safe_load(content) + except yaml.YAMLError as ex: + raise LoadTemplateError("Invalid YAML: {}".format(str(ex))) + if isinstance(loaded, str): + return Template(name=name, prompt=loaded) + loaded["name"] = name + try: + return Template(**loaded) + except pydantic.ValidationError as ex: + msg = "A validation error occurred:\n" + msg += render_errors(ex.errors()) + raise LoadTemplateError(msg) + + +def load_template(name: str) -> Template: + "Load template, or raise LoadTemplateError(msg)" + if name.startswith("https://") or name.startswith("http://"): + response = httpx.get(name) + try: + response.raise_for_status() + except httpx.HTTPStatusError as ex: + raise LoadTemplateError("Could not load template {}: {}".format(name, ex)) + return _parse_yaml_template(name, response.text) + + potential_path = pathlib.Path(name) + + if has_plugin_prefix(name) and not potential_path.exists(): + prefix, rest = name.split(":", 1) + loaders = get_template_loaders() + if prefix not in loaders: + raise LoadTemplateError("Unknown template prefix: {}".format(prefix)) + loader = loaders[prefix] + try: + return loader(rest) + except Exception as ex: + raise LoadTemplateError("Could not load template {}: {}".format(name, ex)) + + # Try local file + if potential_path.exists(): + path = potential_path + else: + # Look for template in template_dir() + path = template_dir() / f"{name}.yaml" + if not path.exists(): + raise LoadTemplateError(f"Invalid template: {name}") + content = path.read_text() + template_obj = _parse_yaml_template(name, content) + # We trust functions here because they came from the filesystem + template_obj._functions_is_trusted = True + return template_obj + + +def _tools_from_code(code_or_path: str) -> List[Tool]: + """ + Treat all Python functions in the code as tools + """ + if "\n" not in code_or_path and code_or_path.endswith(".py"): + try: + code_or_path = pathlib.Path(code_or_path).read_text() + except FileNotFoundError: + raise click.ClickException("File not found: {}".format(code_or_path)) + namespace: Dict[str, Any] = {} + tools = [] + try: + exec(code_or_path, namespace) + except SyntaxError as ex: + raise click.ClickException("Error in --functions definition: {}".format(ex)) + # Register all callables in the locals dict: + for name, value in namespace.items(): + if callable(value) and not name.startswith("_"): + tools.append(Tool.function(value)) + return tools + + +def _debug_tool_call(_, tool_call, tool_result): + click.echo( + click.style( + "\nTool call: {}({})".format(tool_call.name, tool_call.arguments), + fg="yellow", + bold=True, + ), + err=True, + ) + output = "" + attachments = "" + if tool_result.attachments: + attachments += "\nAttachments:\n" + for attachment in tool_result.attachments: + attachments += f" {repr(attachment)}\n" + + try: + output = json.dumps(json.loads(tool_result.output), indent=2) + except ValueError: + output = tool_result.output + output += attachments + click.echo( + click.style( + textwrap.indent(output, " ") + ("\n" if not tool_result.exception else ""), + fg="green", + bold=True, + ), + err=True, + ) + if tool_result.exception: + click.echo( + click.style( + " Exception: {}".format(tool_result.exception), + fg="red", + bold=True, + ), + err=True, + ) + + +def _approve_tool_call(_, tool_call): + click.echo( + click.style( + "Tool call: {}({})".format(tool_call.name, tool_call.arguments), + fg="yellow", + bold=True, + ), + err=True, + ) + if not click.confirm("Approve tool call?"): + raise CancelToolCall("User cancelled tool call") + + +def _gather_tools( + tool_specs: List[str], python_tools: List[str] +) -> List[Union[Tool, Type[Toolbox]]]: + tools: List[Union[Tool, Type[Toolbox]]] = [] + if python_tools: + for code_or_path in python_tools: + tools.extend(_tools_from_code(code_or_path)) + registered_tools = get_tools() + registered_classes = dict( + (key, value) + for key, value in registered_tools.items() + if inspect.isclass(value) + ) + bad_tools = [ + tool for tool in tool_specs if tool.split("(")[0] not in registered_tools + ] + if bad_tools: + raise click.ClickException( + "Tool(s) {} not found. Available tools: {}".format( + ", ".join(bad_tools), ", ".join(registered_tools.keys()) + ) + ) + for tool_spec in tool_specs: + if not tool_spec[0].isupper(): + # It's a function + tools.append(registered_tools[tool_spec]) + else: + # It's a class + tools.append(instantiate_from_spec(registered_classes, tool_spec)) + return tools + + +def _get_conversation_tools(conversation, tools): + if conversation and not tools and conversation.responses: + # Copy plugin tools from first response in conversation + initial_tools = conversation.responses[0].prompt.tools + if initial_tools: + # Only tools from plugins: + return [tool.name for tool in initial_tools if tool.plugin] diff --git a/build/lib/llm/default_plugins/__init__.py b/build/lib/llm/default_plugins/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/build/lib/llm/default_plugins/default_tools.py b/build/lib/llm/default_plugins/default_tools.py new file mode 100644 index 000000000..53ff72cd0 --- /dev/null +++ b/build/lib/llm/default_plugins/default_tools.py @@ -0,0 +1,8 @@ +import llm +from llm.tools import llm_time, llm_version + + +@llm.hookimpl +def register_tools(register): + register(llm_version) + register(llm_time) diff --git a/build/lib/llm/default_plugins/openai_models.py b/build/lib/llm/default_plugins/openai_models.py new file mode 100644 index 000000000..9a7013390 --- /dev/null +++ b/build/lib/llm/default_plugins/openai_models.py @@ -0,0 +1,1212 @@ +from llm import ( + AsyncConversation, + AsyncKeyModel, + AsyncResponse, + Conversation, + EmbeddingModel, + KeyModel, + Prompt, + Response, + StreamEvent, + hookimpl, +) +import llm +from llm.utils import ( + dicts_to_table_string, + remove_dict_none_values, + logging_client, + simplify_usage_dict, +) +import click +import datetime +from enum import Enum +import httpx +import openai +import os + +from pydantic import field_validator, Field + +from typing import AsyncGenerator, cast, List, Iterable, Iterator, Optional, Union +import json +import yaml + + +@hookimpl +def register_models(register): + # GPT-4o + register( + Chat("gpt-4o", vision=True, supports_schema=True, supports_tools=True), + AsyncChat("gpt-4o", vision=True, supports_schema=True, supports_tools=True), + aliases=("4o",), + ) + register( + Chat("chatgpt-4o-latest", vision=True), + AsyncChat("chatgpt-4o-latest", vision=True), + aliases=("chatgpt-4o",), + ) + register( + Chat("gpt-4o-mini", vision=True, supports_schema=True, supports_tools=True), + AsyncChat( + "gpt-4o-mini", vision=True, supports_schema=True, supports_tools=True + ), + aliases=("4o-mini",), + ) + for audio_model_id in ( + "gpt-4o-audio-preview", + "gpt-4o-audio-preview-2024-12-17", + "gpt-4o-audio-preview-2024-10-01", + "gpt-4o-mini-audio-preview", + "gpt-4o-mini-audio-preview-2024-12-17", + ): + register( + Chat(audio_model_id, audio=True), + AsyncChat(audio_model_id, audio=True), + ) + # GPT-4.1 + for model_id in ("gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano"): + register( + Chat(model_id, vision=True, supports_schema=True, supports_tools=True), + AsyncChat(model_id, vision=True, supports_schema=True, supports_tools=True), + aliases=(model_id.replace("gpt-", ""),), + ) + # 3.5 and 4 + register( + Chat("gpt-3.5-turbo"), AsyncChat("gpt-3.5-turbo"), aliases=("3.5", "chatgpt") + ) + register( + Chat("gpt-3.5-turbo-16k"), + AsyncChat("gpt-3.5-turbo-16k"), + aliases=("chatgpt-16k", "3.5-16k"), + ) + register(Chat("gpt-4"), AsyncChat("gpt-4"), aliases=("4", "gpt4")) + register(Chat("gpt-4-32k"), AsyncChat("gpt-4-32k"), aliases=("4-32k",)) + # GPT-4 Turbo models + register(Chat("gpt-4-1106-preview"), AsyncChat("gpt-4-1106-preview")) + register(Chat("gpt-4-0125-preview"), AsyncChat("gpt-4-0125-preview")) + register(Chat("gpt-4-turbo-2024-04-09"), AsyncChat("gpt-4-turbo-2024-04-09")) + register( + Chat("gpt-4-turbo"), + AsyncChat("gpt-4-turbo"), + aliases=("gpt-4-turbo-preview", "4-turbo", "4t"), + ) + # GPT-4.5 + register( + Chat( + "gpt-4.5-preview-2025-02-27", + vision=True, + supports_schema=True, + supports_tools=True, + ), + AsyncChat( + "gpt-4.5-preview-2025-02-27", + vision=True, + supports_schema=True, + supports_tools=True, + ), + ) + register( + Chat("gpt-4.5-preview", vision=True, supports_schema=True, supports_tools=True), + AsyncChat( + "gpt-4.5-preview", vision=True, supports_schema=True, supports_tools=True + ), + aliases=("gpt-4.5",), + ) + # o1 + for model_id in ("o1", "o1-2024-12-17"): + register( + Chat( + model_id, + vision=True, + can_stream=False, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + AsyncChat( + model_id, + vision=True, + can_stream=False, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + ) + + register( + Chat("o1-preview", allows_system_prompt=False), + AsyncChat("o1-preview", allows_system_prompt=False), + ) + register( + Chat("o1-mini", allows_system_prompt=False), + AsyncChat("o1-mini", allows_system_prompt=False), + ) + register( + Chat("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), + AsyncChat("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), + ) + register( + Chat( + "o3", vision=True, reasoning=True, supports_schema=True, supports_tools=True + ), + AsyncChat( + "o3", vision=True, reasoning=True, supports_schema=True, supports_tools=True + ), + ) + register( + Chat( + "o4-mini", + vision=True, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + AsyncChat( + "o4-mini", + vision=True, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + ) + # GPT-5 + for model_id in ( + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-2025-08-07", + "gpt-5-mini-2025-08-07", + "gpt-5-nano-2025-08-07", + ): + register( + Chat( + model_id, + vision=True, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + AsyncChat( + model_id, + vision=True, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + ) + # GPT-5.1 + for model_id in ( + "gpt-5.1", + "gpt-5.1-chat-latest", + ): + register( + Chat( + model_id, + vision=True, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + AsyncChat( + model_id, + vision=True, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + ) + # GPT-5.2 + for model_id in ("gpt-5.2", "gpt-5.2-chat-latest"): + register( + Chat( + model_id, + vision=True, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + AsyncChat( + model_id, + vision=True, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + ) + # "gpt-5.2-pro" is Responses API only + + # GPT-5.4 + for model_id in ( + "gpt-5.4", + "gpt-5.4-2026-03-05", + "gpt-5.4-mini", + "gpt-5.4-mini-2026-03-17", + "gpt-5.4-nano", + "gpt-5.4-nano-2026-03-17", + ): + register( + Chat( + model_id, + vision=True, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + AsyncChat( + model_id, + vision=True, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + ) + + # The -instruct completion model + register( + Completion("gpt-3.5-turbo-instruct", default_max_tokens=256), + aliases=("3.5-instruct", "chatgpt-instruct"), + ) + + # Load extra models + extra_path = llm.user_dir() / "extra-openai-models.yaml" + if not extra_path.exists(): + return + with open(extra_path) as f: + extra_models = yaml.safe_load(f) + for extra_model in extra_models: + model_id = extra_model["model_id"] + aliases = extra_model.get("aliases", []) + model_name = extra_model["model_name"] + api_base = extra_model.get("api_base") + api_type = extra_model.get("api_type") + api_version = extra_model.get("api_version") + api_engine = extra_model.get("api_engine") + headers = extra_model.get("headers") + reasoning = extra_model.get("reasoning") + kwargs = {} + if extra_model.get("can_stream") is False: + kwargs["can_stream"] = False + if extra_model.get("supports_schema") is True: + kwargs["supports_schema"] = True + if extra_model.get("supports_tools") is True: + kwargs["supports_tools"] = True + if extra_model.get("vision") is True: + kwargs["vision"] = True + if extra_model.get("audio") is True: + kwargs["audio"] = True + if extra_model.get("completion"): + klass = Completion + async_klass = None + else: + klass = Chat + async_klass = AsyncChat + model_kwargs = dict( + model_id=model_id, + model_name=model_name, + api_base=api_base, + api_type=api_type, + api_version=api_version, + api_engine=api_engine, + headers=headers, + reasoning=reasoning, + **kwargs, + ) + chat_model = klass(**model_kwargs) + async_model = async_klass(**model_kwargs) if async_klass else None + if api_base: + chat_model.needs_key = None + if async_model: + async_model.needs_key = None + if extra_model.get("api_key_name"): + chat_model.needs_key = extra_model["api_key_name"] + if async_model: + async_model.needs_key = extra_model["api_key_name"] + register( + chat_model, + async_model, + aliases=aliases, + ) + + +@hookimpl +def register_embedding_models(register): + register( + OpenAIEmbeddingModel("text-embedding-ada-002", "text-embedding-ada-002"), + aliases=( + "ada", + "ada-002", + ), + ) + register( + OpenAIEmbeddingModel("text-embedding-3-small", "text-embedding-3-small"), + aliases=("3-small",), + ) + register( + OpenAIEmbeddingModel("text-embedding-3-large", "text-embedding-3-large"), + aliases=("3-large",), + ) + # With varying dimensions + register( + OpenAIEmbeddingModel( + "text-embedding-3-small-512", "text-embedding-3-small", 512 + ), + aliases=("3-small-512",), + ) + register( + OpenAIEmbeddingModel( + "text-embedding-3-large-256", "text-embedding-3-large", 256 + ), + aliases=("3-large-256",), + ) + register( + OpenAIEmbeddingModel( + "text-embedding-3-large-1024", "text-embedding-3-large", 1024 + ), + aliases=("3-large-1024",), + ) + + +class OpenAIEmbeddingModel(EmbeddingModel): + needs_key = "openai" + key_env_var = "OPENAI_API_KEY" + batch_size = 100 + + def __init__(self, model_id, openai_model_id, dimensions=None): + self.model_id = model_id + self.openai_model_id = openai_model_id + self.dimensions = dimensions + + def embed_batch(self, items: Iterable[Union[str, bytes]]) -> Iterator[List[float]]: + kwargs = { + "input": items, + "model": self.openai_model_id, + } + if self.dimensions: + kwargs["dimensions"] = self.dimensions + client = openai.OpenAI(api_key=self.get_key()) + results = client.embeddings.create(**kwargs).data + return ([float(r) for r in result.embedding] for result in results) + + +@hookimpl +def register_commands(cli): + @cli.group(name="openai") + def openai_(): + "Commands for working directly with the OpenAI API" + + @openai_.command() + @click.option("json_", "--json", is_flag=True, help="Output as JSON") + @click.option("--key", help="OpenAI API key") + def models(json_, key): + "List models available to you from the OpenAI API" + from llm import get_key + + api_key = get_key(key, "openai", "OPENAI_API_KEY") + response = httpx.get( + "https://api.openai.com/v1/models", + headers={"Authorization": f"Bearer {api_key}"}, + ) + if response.status_code != 200: + raise click.ClickException( + f"Error {response.status_code} from OpenAI API: {response.text}" + ) + models = response.json()["data"] + if json_: + click.echo(json.dumps(models, indent=4)) + else: + to_print = [] + for model in models: + # Print id, owned_by, root, created as ISO 8601 + created_str = datetime.datetime.fromtimestamp( + model["created"], datetime.timezone.utc + ).isoformat() + to_print.append( + { + "id": model["id"], + "owned_by": model["owned_by"], + "created": created_str, + } + ) + done = dicts_to_table_string("id owned_by created".split(), to_print) + print("\n".join(done)) + + +class SharedOptions(llm.Options): + temperature: Optional[float] = Field( + description=( + "What sampling temperature to use, between 0 and 2. Higher values like " + "0.8 will make the output more random, while lower values like 0.2 will " + "make it more focused and deterministic." + ), + ge=0, + le=2, + default=None, + ) + max_tokens: Optional[int] = Field( + description="Maximum number of tokens to generate.", default=None + ) + top_p: Optional[float] = Field( + description=( + "An alternative to sampling with temperature, called nucleus sampling, " + "where the model considers the results of the tokens with top_p " + "probability mass. So 0.1 means only the tokens comprising the top " + "10% probability mass are considered. Recommended to use top_p or " + "temperature but not both." + ), + ge=0, + le=1, + default=None, + ) + frequency_penalty: Optional[float] = Field( + description=( + "Number between -2.0 and 2.0. Positive values penalize new tokens based " + "on their existing frequency in the text so far, decreasing the model's " + "likelihood to repeat the same line verbatim." + ), + ge=-2, + le=2, + default=None, + ) + presence_penalty: Optional[float] = Field( + description=( + "Number between -2.0 and 2.0. Positive values penalize new tokens based " + "on whether they appear in the text so far, increasing the model's " + "likelihood to talk about new topics." + ), + ge=-2, + le=2, + default=None, + ) + stop: Optional[str] = Field( + description=("A string where the API will stop generating further tokens."), + default=None, + ) + logit_bias: Optional[Union[dict, str]] = Field( + description=( + "Modify the likelihood of specified tokens appearing in the completion. " + 'Pass a JSON string like \'{"1712":-100, "892":-100, "1489":-100}\'' + ), + default=None, + ) + seed: Optional[int] = Field( + description="Integer seed to attempt to sample deterministically", + default=None, + ) + + @field_validator("logit_bias") + def validate_logit_bias(cls, logit_bias): + if logit_bias is None: + return None + + if isinstance(logit_bias, str): + try: + logit_bias = json.loads(logit_bias) + except json.JSONDecodeError: + raise ValueError("Invalid JSON in logit_bias string") + + validated_logit_bias = {} + for key, value in logit_bias.items(): + try: + int_key = int(key) + int_value = int(value) + if -100 <= int_value <= 100: + validated_logit_bias[int_key] = int_value + else: + raise ValueError("Value must be between -100 and 100") + except ValueError: + raise ValueError("Invalid key-value pair in logit_bias dictionary") + + return validated_logit_bias + + +class ReasoningEffortEnum(str, Enum): + none = "none" + minimal = "minimal" + low = "low" + medium = "medium" + high = "high" + xhigh = "xhigh" + + +class OptionsForReasoning(SharedOptions): + json_object: Optional[bool] = Field( + description="Output a valid JSON object {...}. Prompt must mention JSON.", + default=None, + ) + reasoning_effort: Optional[ReasoningEffortEnum] = Field( + description=( + "Constraints effort on reasoning for reasoning models. Currently supported " + "values are low, medium, and high. Reducing reasoning effort can result in " + "faster responses and fewer tokens used on reasoning in a response." + ), + default=None, + ) + + +def _attachment(attachment): + url = attachment.url + base64_content = "" + if not url or attachment.resolve_type().startswith("audio/"): + base64_content = attachment.base64_content() + url = f"data:{attachment.resolve_type()};base64,{base64_content}" + if attachment.resolve_type() == "application/pdf": + if not base64_content: + base64_content = attachment.base64_content() + return { + "type": "file", + "file": { + "filename": f"{attachment.id()}.pdf", + "file_data": f"data:application/pdf;base64,{base64_content}", + }, + } + if attachment.resolve_type().startswith("image/"): + return {"type": "image_url", "image_url": {"url": url}} + else: + format_ = "wav" if attachment.resolve_type() == "audio/wav" else "mp3" + return { + "type": "input_audio", + "input_audio": { + "data": base64_content, + "format": format_, + }, + } + + +class _Shared: + def __init__( + self, + model_id, + key=None, + model_name=None, + api_base=None, + api_type=None, + api_version=None, + api_engine=None, + headers=None, + can_stream=True, + vision=False, + audio=False, + reasoning=False, + supports_schema=False, + supports_tools=False, + allows_system_prompt=True, + ): + self.model_id = model_id + self.key = key + self.supports_schema = supports_schema + self.supports_tools = supports_tools + self.model_name = model_name + self.api_base = api_base + self.api_type = api_type + self.api_version = api_version + self.api_engine = api_engine + self.headers = headers + self.can_stream = can_stream + self.vision = vision + self.allows_system_prompt = allows_system_prompt + + self.attachment_types = set() + + if reasoning: + self.Options = OptionsForReasoning + + if vision: + self.attachment_types.update( + { + "image/png", + "image/jpeg", + "image/webp", + "image/gif", + "application/pdf", + } + ) + + if audio: + self.attachment_types.update( + { + "audio/wav", + "audio/mpeg", + } + ) + + def __str__(self) -> str: + return "OpenAI Chat: {}".format(self.model_id) + + def _append_llm_message(self, out, message, current_system): + """Translate one llm.Message into one (or more) OpenAI message + dicts and append them to ``out``. + + Returns the (possibly updated) current_system value so the caller + can dedup consecutive identical system messages. + """ + from llm.parts import ( + AttachmentPart, + TextPart, + ToolCallPart, + ToolResultPart, + ) + + text_bits = [] + attachment_items = [] + tool_calls = [] + tool_results = [] + + for part in message.parts: + if isinstance(part, TextPart): + text_bits.append(part.text) + elif isinstance(part, AttachmentPart) and part.attachment: + attachment_items.append(_attachment(part.attachment)) + elif isinstance(part, ToolCallPart): + tool_calls.append( + { + "type": "function", + "id": part.tool_call_id, + "function": { + "name": part.name, + "arguments": json.dumps(part.arguments), + }, + } + ) + elif isinstance(part, ToolResultPart): + tool_results.append( + { + "role": "tool", + "tool_call_id": part.tool_call_id, + "content": part.output, + } + ) + + # Role "tool" emits one OpenAI "tool" message per ToolResultPart. + if message.role == "tool": + out.extend(tool_results) + return current_system + + # System dedup — skip if we just emitted this exact system text. + if message.role == "system": + text = "".join(text_bits) + if text == current_system: + return current_system + current_system = text + + if attachment_items: + content = [] + if text_bits: + content.append({"type": "text", "text": "".join(text_bits)}) + content.extend(attachment_items) + entry = {"role": message.role, "content": content} + else: + entry = { + "role": message.role, + "content": "".join(text_bits) if text_bits else None, + } + + if tool_calls: + entry["tool_calls"] = tool_calls + # OpenAI expects content=null when only tool_calls are present. + if not text_bits: + entry["content"] = None + elif entry["content"] is None and message.role != "assistant": + # For user/system, an empty message is pointless — drop it. + return current_system + + out.append(entry) + return current_system + + def build_messages(self, prompt, conversation): + """Translate prompt.messages into OpenAI's wire format. + + Under the Phase 7 invariant, ``prompt.messages`` is the full + chain for this turn — Conversation.prompt and response.reply + pre-bake the history into it. The ``conversation`` parameter + is unused and retained only for the plugin API contract. + """ + messages: List[Dict[str, Any]] = [] + current_system: Optional[str] = None + for msg in prompt.messages: + current_system = self._append_llm_message( + messages, msg, current_system + ) + return messages + + def set_usage(self, response, usage): + if not usage: + return + input_tokens = usage.pop("prompt_tokens") + output_tokens = usage.pop("completion_tokens") + usage.pop("total_tokens") + response.set_usage( + input=input_tokens, output=output_tokens, details=simplify_usage_dict(usage) + ) + + def get_client(self, key, *, async_=False): + kwargs = {} + if self.api_base: + kwargs["base_url"] = self.api_base + if self.api_type: + kwargs["api_type"] = self.api_type + if self.api_version: + kwargs["api_version"] = self.api_version + if self.api_engine: + kwargs["engine"] = self.api_engine + if self.needs_key: + kwargs["api_key"] = self.get_key(key) + else: + # OpenAI-compatible models don't need a key, but the + # openai client library requires one + kwargs["api_key"] = "DUMMY_KEY" + if self.headers: + kwargs["default_headers"] = self.headers + if os.environ.get("LLM_OPENAI_SHOW_RESPONSES"): + kwargs["http_client"] = logging_client() + if async_: + return openai.AsyncOpenAI(**kwargs) + else: + return openai.OpenAI(**kwargs) + + def build_kwargs(self, prompt, stream): + kwargs = dict(not_nulls(prompt.options)) + json_object = kwargs.pop("json_object", None) + if "max_tokens" not in kwargs and self.default_max_tokens is not None: + kwargs["max_tokens"] = self.default_max_tokens + if json_object: + kwargs["response_format"] = {"type": "json_object"} + if prompt.schema: + kwargs["response_format"] = { + "type": "json_schema", + "json_schema": {"name": "output", "schema": prompt.schema}, + } + if prompt.tools: + kwargs["tools"] = [ + { + "type": "function", + "function": { + "name": tool.name, + "description": tool.description or None, + "parameters": tool.input_schema, + }, + } + for tool in prompt.tools + ] + if stream: + kwargs["stream_options"] = {"include_usage": True} + return kwargs + + +class Chat(_Shared, KeyModel): + needs_key = "openai" + key_env_var = "OPENAI_API_KEY" + default_max_tokens = None + + class Options(SharedOptions): + json_object: Optional[bool] = Field( + description="Output a valid JSON object {...}. Prompt must mention JSON.", + default=None, + ) + + def execute( + self, + prompt: Prompt, + stream: bool, + response: Response, + conversation: Optional[Conversation] = None, + key: Optional[str] = None, + ) -> Iterator[str]: + if prompt.system and not self.allows_system_prompt: + raise NotImplementedError("Model does not support system prompts") + messages = self.build_messages(prompt, conversation) + kwargs = self.build_kwargs(prompt, stream) + client = self.get_client(key) + usage = None + if stream: + completion = client.chat.completions.create( + model=self.model_name or self.model_id, + messages=messages, + stream=True, + **kwargs, + ) + chunks = [] + tool_calls = {} + # part_index allocator. Text always uses 0. Each tool call + # at delta index i is assigned a part_index past any text + # that was seen, so _build_parts groups them correctly. + seen_text = False + tc_part_index = {} + next_part_index = 1 + for chunk in completion: + chunks.append(chunk) + if chunk.usage: + usage = chunk.usage.model_dump() + if chunk.choices and chunk.choices[0].delta: + for tool_call in chunk.choices[0].delta.tool_calls or []: + if tool_call.function.arguments is None: + tool_call.function.arguments = "" + idx = tool_call.index + if idx not in tool_calls: + tool_calls[idx] = tool_call + tc_part_index[idx] = next_part_index + next_part_index += 1 + yield StreamEvent( + type="tool_call_name", + chunk=tool_call.function.name or "", + part_index=tc_part_index[idx], + tool_call_id=tool_call.id, + ) + else: + tool_calls[idx].function.arguments += ( + tool_call.function.arguments + ) + if tool_call.function.arguments: + yield StreamEvent( + type="tool_call_args", + chunk=tool_call.function.arguments, + part_index=tc_part_index[idx], + tool_call_id=tool_calls[idx].id, + ) + try: + content = chunk.choices[0].delta.content + except IndexError: + content = None + if content: + # Empty strings are noise (OpenAI's first chunk + # with role=assistant has content=""). + seen_text = True + yield StreamEvent( + type="text", chunk=content, part_index=0 + ) + response.response_json = remove_dict_none_values(combine_chunks(chunks)) + if tool_calls: + for value in tool_calls.values(): + response.add_tool_call( + llm.ToolCall( + tool_call_id=value.id, + name=value.function.name, + arguments=json.loads(value.function.arguments), + ) + ) + else: + completion = client.chat.completions.create( + model=self.model_name or self.model_id, + messages=messages, + stream=False, + **kwargs, + ) + usage = completion.usage.model_dump() + response.response_json = remove_dict_none_values(completion.model_dump()) + part_index = 0 + for tool_call in completion.choices[0].message.tool_calls or []: + response.add_tool_call( + llm.ToolCall( + tool_call_id=tool_call.id, + name=tool_call.function.name, + arguments=json.loads(tool_call.function.arguments), + ) + ) + part_index += 1 + yield StreamEvent( + type="tool_call_name", + chunk=tool_call.function.name or "", + part_index=part_index, + tool_call_id=tool_call.id, + ) + yield StreamEvent( + type="tool_call_args", + chunk=tool_call.function.arguments or "", + part_index=part_index, + tool_call_id=tool_call.id, + ) + if completion.choices[0].message.content is not None: + yield StreamEvent( + type="text", + chunk=completion.choices[0].message.content, + part_index=0, + ) + # Capture the reasoning token count BEFORE set_usage runs — + # set_usage pops top-level keys and passes the rest through + # simplify_usage_dict, which strips zero-valued entries. + if usage: + reasoning_tokens = ( + (usage.get("completion_tokens_details") or {}).get( + "reasoning_tokens", 0 + ) + ) + if reasoning_tokens: + response._reasoning_token_count = reasoning_tokens + self.set_usage(response, usage) + response._prompt_json = redact_data({"messages": messages}) + + +class AsyncChat(_Shared, AsyncKeyModel): + needs_key = "openai" + key_env_var = "OPENAI_API_KEY" + default_max_tokens = None + + class Options(SharedOptions): + json_object: Optional[bool] = Field( + description="Output a valid JSON object {...}. Prompt must mention JSON.", + default=None, + ) + + async def execute( + self, + prompt: Prompt, + stream: bool, + response: AsyncResponse, + conversation: Optional[AsyncConversation] = None, + key: Optional[str] = None, + ) -> AsyncGenerator[str, None]: + if prompt.system and not self.allows_system_prompt: + raise NotImplementedError("Model does not support system prompts") + messages = self.build_messages(prompt, conversation) + kwargs = self.build_kwargs(prompt, stream) + client = self.get_client(key, async_=True) + usage = None + if stream: + completion = await client.chat.completions.create( + model=self.model_name or self.model_id, + messages=messages, + stream=True, + **kwargs, + ) + chunks = [] + tool_calls = {} + tc_part_index = {} + next_part_index = 1 + async for chunk in completion: + if chunk.usage: + usage = chunk.usage.model_dump() + chunks.append(chunk) + if chunk.choices and chunk.choices[0].delta: + for tool_call in chunk.choices[0].delta.tool_calls or []: + if tool_call.function.arguments is None: + tool_call.function.arguments = "" + idx = tool_call.index + if idx not in tool_calls: + tool_calls[idx] = tool_call + tc_part_index[idx] = next_part_index + next_part_index += 1 + yield StreamEvent( + type="tool_call_name", + chunk=tool_call.function.name or "", + part_index=tc_part_index[idx], + tool_call_id=tool_call.id, + ) + else: + tool_calls[idx].function.arguments += ( + tool_call.function.arguments + ) + if tool_call.function.arguments: + yield StreamEvent( + type="tool_call_args", + chunk=tool_call.function.arguments, + part_index=tc_part_index[idx], + tool_call_id=tool_calls[idx].id, + ) + try: + content = chunk.choices[0].delta.content + except IndexError: + content = None + if content: + yield StreamEvent( + type="text", chunk=content, part_index=0 + ) + if tool_calls: + for value in tool_calls.values(): + response.add_tool_call( + llm.ToolCall( + tool_call_id=value.id, + name=value.function.name, + arguments=json.loads(value.function.arguments), + ) + ) + response.response_json = remove_dict_none_values(combine_chunks(chunks)) + else: + completion = await client.chat.completions.create( + model=self.model_name or self.model_id, + messages=messages, + stream=False, + **kwargs, + ) + response.response_json = remove_dict_none_values(completion.model_dump()) + usage = completion.usage.model_dump() + part_index = 0 + for tool_call in completion.choices[0].message.tool_calls or []: + response.add_tool_call( + llm.ToolCall( + tool_call_id=tool_call.id, + name=tool_call.function.name, + arguments=json.loads(tool_call.function.arguments), + ) + ) + part_index += 1 + yield StreamEvent( + type="tool_call_name", + chunk=tool_call.function.name or "", + part_index=part_index, + tool_call_id=tool_call.id, + ) + yield StreamEvent( + type="tool_call_args", + chunk=tool_call.function.arguments or "", + part_index=part_index, + tool_call_id=tool_call.id, + ) + if completion.choices[0].message.content is not None: + yield StreamEvent( + type="text", + chunk=completion.choices[0].message.content, + part_index=0, + ) + # See sync Chat.execute: capture reasoning before set_usage mutates. + if usage: + reasoning_tokens = ( + (usage.get("completion_tokens_details") or {}).get( + "reasoning_tokens", 0 + ) + ) + if reasoning_tokens: + response._reasoning_token_count = reasoning_tokens + self.set_usage(response, usage) + response._prompt_json = redact_data({"messages": messages}) + + +class Completion(Chat): + class Options(SharedOptions): + logprobs: Optional[int] = Field( + description="Include the log probabilities of most likely N per token", + default=None, + le=5, + ) + + def __init__(self, *args, default_max_tokens=None, **kwargs): + super().__init__(*args, **kwargs) + self.default_max_tokens = default_max_tokens + + def __str__(self) -> str: + return "OpenAI Completion: {}".format(self.model_id) + + def execute( + self, + prompt: Prompt, + stream: bool, + response: Response, + conversation: Optional[Conversation] = None, + key: Optional[str] = None, + ) -> Iterator[str]: + if prompt.system: + raise NotImplementedError( + "System prompts are not supported for OpenAI completion models" + ) + messages = [] + if conversation is not None: + for prev_response in conversation.responses: + messages.append(prev_response.prompt.prompt) + messages.append(cast(Response, prev_response).text()) + messages.append(prompt.prompt) + kwargs = self.build_kwargs(prompt, stream) + client = self.get_client(key) + if stream: + completion = client.completions.create( + model=self.model_name or self.model_id, + prompt="\n".join(messages), + stream=True, + **kwargs, + ) + chunks = [] + for chunk in completion: + chunks.append(chunk) + try: + content = chunk.choices[0].text + except IndexError: + content = None + if content is not None: + yield content + combined = combine_chunks(chunks) + cleaned = remove_dict_none_values(combined) + response.response_json = cleaned + else: + completion = client.completions.create( + model=self.model_name or self.model_id, + prompt="\n".join(messages), + stream=False, + **kwargs, + ) + response.response_json = remove_dict_none_values(completion.model_dump()) + yield completion.choices[0].text + response._prompt_json = redact_data({"messages": messages}) + + +def not_nulls(data) -> dict: + return {key: value for key, value in data if value is not None} + + +def combine_chunks(chunks: List) -> dict: + content = "" + role = None + finish_reason = None + # If any of them have log probability, we're going to persist + # those later on + logprobs = [] + usage = {} + + for item in chunks: + if item.usage: + usage = item.usage.model_dump() + for choice in item.choices: + if choice.logprobs and hasattr(choice.logprobs, "top_logprobs"): + logprobs.append( + { + "text": choice.text if hasattr(choice, "text") else None, + "top_logprobs": choice.logprobs.top_logprobs, + } + ) + + if not hasattr(choice, "delta"): + content += choice.text + continue + role = choice.delta.role + if choice.delta.content is not None: + content += choice.delta.content + if choice.finish_reason is not None: + finish_reason = choice.finish_reason + + # Imitations of the OpenAI API may be missing some of these fields + combined = { + "content": content, + "role": role, + "finish_reason": finish_reason, + "usage": usage, + } + if logprobs: + combined["logprobs"] = logprobs + if chunks: + for key in ("id", "object", "model", "created", "index"): + value = getattr(chunks[0], key, None) + if value is not None: + combined[key] = value + + return combined + + +def redact_data(input_dict): + """ + Recursively search through the input dictionary for any 'image_url' keys + and modify the 'url' value to be just 'data:...'. + + Also redact input_audio.data keys + """ + if isinstance(input_dict, dict): + for key, value in input_dict.items(): + if ( + key == "image_url" + and isinstance(value, dict) + and "url" in value + and value["url"].startswith("data:") + ): + value["url"] = "data:..." + elif key == "input_audio" and isinstance(value, dict) and "data" in value: + value["data"] = "..." + else: + redact_data(value) + elif isinstance(input_dict, list): + for item in input_dict: + redact_data(item) + return input_dict diff --git a/build/lib/llm/embeddings.py b/build/lib/llm/embeddings.py new file mode 100644 index 000000000..90b983a11 --- /dev/null +++ b/build/lib/llm/embeddings.py @@ -0,0 +1,367 @@ +from .models import EmbeddingModel +from .embeddings_migrations import embeddings_migrations +from dataclasses import dataclass +import hashlib +from itertools import islice +import json +from sqlite_utils import Database +from sqlite_utils.db import Table +import time +from typing import cast, Any, Dict, Iterable, List, Optional, Tuple, Union + + +@dataclass +class Entry: + id: str + score: Optional[float] + content: Optional[str] = None + metadata: Optional[Dict[str, Any]] = None + + +class Collection: + class DoesNotExist(Exception): + pass + + def __init__( + self, + name: str, + db: Optional[Database] = None, + *, + model: Optional[EmbeddingModel] = None, + model_id: Optional[str] = None, + create: bool = True, + ) -> None: + """ + A collection of embeddings + + Returns the collection with the given name, creating it if it does not exist. + + If you set create=False a Collection.DoesNotExist exception will be raised if the + collection does not already exist. + + Args: + db (sqlite_utils.Database): Database to store the collection in + name (str): Name of the collection + model (llm.models.EmbeddingModel, optional): Embedding model to use + model_id (str, optional): Alternatively, ID of the embedding model to use + create (bool, optional): Whether to create the collection if it does not exist + """ + import llm + + self.db = db or Database(memory=True) + self.name = name + self._model = model + + embeddings_migrations.apply(self.db) + + rows = list(self.db["collections"].rows_where("name = ?", [self.name])) + if rows: + row = rows[0] + self.id = row["id"] + self.model_id = row["model"] + else: + if create: + # Collection does not exist, so model or model_id is required + if not model and not model_id: + raise ValueError( + "Either model= or model_id= must be provided when creating a new collection" + ) + # Create it + if model_id: + # Resolve alias + model = llm.get_embedding_model(model_id) + self._model = model + model_id = cast(EmbeddingModel, model).model_id + self.id = ( + cast(Table, self.db["collections"]) + .insert( + { + "name": self.name, + "model": model_id, + } + ) + .last_pk + ) + else: + raise self.DoesNotExist(f"Collection '{name}' does not exist") + + def model(self) -> EmbeddingModel: + "Return the embedding model used by this collection" + import llm + + if self._model is None: + self._model = llm.get_embedding_model(self.model_id) + + return cast(EmbeddingModel, self._model) + + def count(self) -> int: + """ + Count the number of items in the collection. + + Returns: + int: Number of items in the collection + """ + return next( + self.db.query( + """ + select count(*) as c from embeddings where collection_id = ( + select id from collections where name = ? + ) + """, + (self.name,), + ) + )["c"] + + def embed( + self, + id: str, + value: Union[str, bytes], + metadata: Optional[Dict[str, Any]] = None, + store: bool = False, + ) -> None: + """ + Embed value and store it in the collection with a given ID. + + Args: + id (str): ID for the value + value (str or bytes): value to be embedded + metadata (dict, optional): Metadata to be stored + store (bool, optional): Whether to store the value in the content or content_blob column + """ + from llm import encode + + content_hash = self.content_hash(value) + if self.db["embeddings"].count_where( + "content_hash = ? and collection_id = ?", [content_hash, self.id] + ): + return + embedding = self.model().embed(value) + cast(Table, self.db["embeddings"]).insert( + { + "collection_id": self.id, + "id": id, + "embedding": encode(embedding), + "content": value if (store and isinstance(value, str)) else None, + "content_blob": value if (store and isinstance(value, bytes)) else None, + "content_hash": content_hash, + "metadata": json.dumps(metadata) if metadata else None, + "updated": int(time.time()), + }, + replace=True, + ) + + def embed_multi( + self, + entries: Iterable[Tuple[str, Union[str, bytes]]], + store: bool = False, + batch_size: int = 100, + ) -> None: + """ + Embed multiple texts and store them in the collection with given IDs. + + Args: + entries (iterable): Iterable of (id: str, text: str) tuples + store (bool, optional): Whether to store the text in the content column + batch_size (int, optional): custom maximum batch size to use + """ + self.embed_multi_with_metadata( + ((id, value, None) for id, value in entries), + store=store, + batch_size=batch_size, + ) + + def embed_multi_with_metadata( + self, + entries: Iterable[Tuple[str, Union[str, bytes], Optional[Dict[str, Any]]]], + store: bool = False, + batch_size: int = 100, + ) -> None: + """ + Embed multiple values along with metadata and store them in the collection with given IDs. + + Args: + entries (iterable): Iterable of (id: str, value: str or bytes, metadata: None or dict) + store (bool, optional): Whether to store the value in the content or content_blob column + batch_size (int, optional): custom maximum batch size to use + """ + import llm + + batch_size = min(batch_size, (self.model().batch_size or batch_size)) + iterator = iter(entries) + collection_id = self.id + while True: + batch = list(islice(iterator, batch_size)) + if not batch: + break + # Calculate hashes first + items_and_hashes = [(item, self.content_hash(item[1])) for item in batch] + # Any of those hashes already exist? + existing_ids = [ + row["id"] + for row in self.db.query( + """ + select id from embeddings + where collection_id = ? and content_hash in ({}) + """.format(",".join("?" for _ in items_and_hashes)), + [collection_id] + + [item_and_hash[1] for item_and_hash in items_and_hashes], + ) + ] + filtered_batch = [item for item in batch if item[0] not in existing_ids] + embeddings = list( + self.model().embed_multi(item[1] for item in filtered_batch) + ) + with self.db.conn: + cast(Table, self.db["embeddings"]).insert_all( + ( + { + "collection_id": collection_id, + "id": id, + "embedding": llm.encode(embedding), + "content": ( + value if (store and isinstance(value, str)) else None + ), + "content_blob": ( + value if (store and isinstance(value, bytes)) else None + ), + "content_hash": self.content_hash(value), + "metadata": json.dumps(metadata) if metadata else None, + "updated": int(time.time()), + } + for (embedding, (id, value, metadata)) in zip( + embeddings, filtered_batch + ) + ), + replace=True, + ) + + def similar_by_vector( + self, + vector: List[float], + number: int = 10, + skip_id: Optional[str] = None, + prefix: Optional[str] = None, + ) -> List[Entry]: + """ + Find similar items in the collection by a given vector. + + Args: + vector (list): Vector to search by + number (int, optional): Number of similar items to return + skip_id (str, optional): An ID to exclude from the results + prefix: (str, optional): Filter results to IDs witih this prefix + + Returns: + list: List of Entry objects + """ + import llm + + def distance_score(other_encoded): + other_vector = llm.decode(other_encoded) + return llm.cosine_similarity(other_vector, vector) + + self.db.register_function(distance_score, replace=True) + + where_bits = ["collection_id = ?"] + where_args = [str(self.id)] + + if prefix: + where_bits.append("id LIKE ? || '%'") + where_args.append(prefix) + + if skip_id: + where_bits.append("id != ?") + where_args.append(skip_id) + + return [ + Entry( + id=row["id"], + score=row["score"], + content=row["content"], + metadata=json.loads(row["metadata"]) if row["metadata"] else None, + ) + for row in self.db.query( + """ + select id, content, metadata, distance_score(embedding) as score + from embeddings + where {where} + order by score desc limit {number} + """.format( + where=" and ".join(where_bits), + number=number, + ), + where_args, + ) + ] + + def similar_by_id( + self, id: str, number: int = 10, prefix: Optional[str] = None + ) -> List[Entry]: + """ + Find similar items in the collection by a given ID. + + Args: + id (str): ID to search by + number (int, optional): Number of similar items to return + prefix: (str, optional): Filter results to IDs with this prefix + + Returns: + list: List of Entry objects + """ + import llm + + matches = list( + self.db["embeddings"].rows_where( + "collection_id = ? and id = ?", (self.id, id) + ) + ) + if not matches: + raise self.DoesNotExist("ID not found") + embedding = matches[0]["embedding"] + comparison_vector = llm.decode(embedding) + return self.similar_by_vector( + comparison_vector, number, skip_id=id, prefix=prefix + ) + + def similar( + self, value: Union[str, bytes], number: int = 10, prefix: Optional[str] = None + ) -> List[Entry]: + """ + Find similar items in the collection by a given value. + + Args: + value (str or bytes): value to search by + number (int, optional): Number of similar items to return + prefix: (str, optional): Filter results to IDs with this prefix + + Returns: + list: List of Entry objects + """ + comparison_vector = self.model().embed(value) + return self.similar_by_vector(comparison_vector, number, prefix=prefix) + + @classmethod + def exists(cls, db: Database, name: str) -> bool: + """ + Does this collection exist in the database? + + Args: + name (str): Name of the collection + """ + rows = list(db["collections"].rows_where("name = ?", [name])) + return bool(rows) + + def delete(self): + """ + Delete the collection and its embeddings from the database + """ + with self.db.conn: + self.db.execute("delete from embeddings where collection_id = ?", [self.id]) + self.db.execute("delete from collections where id = ?", [self.id]) + + @staticmethod + def content_hash(input: Union[str, bytes]) -> bytes: + "Hash content for deduplication. Override to change hashing behavior." + if isinstance(input, str): + input = input.encode("utf8") + return hashlib.md5(input).digest() diff --git a/build/lib/llm/embeddings_migrations.py b/build/lib/llm/embeddings_migrations.py new file mode 100644 index 000000000..69545f3ea --- /dev/null +++ b/build/lib/llm/embeddings_migrations.py @@ -0,0 +1,89 @@ +from sqlite_migrate import Migrations +import hashlib +import time + +embeddings_migrations = Migrations("llm.embeddings") + + +@embeddings_migrations() +def m001_create_tables(db): + db["collections"].create({"id": int, "name": str, "model": str}, pk="id") + db["collections"].create_index(["name"], unique=True) + db["embeddings"].create( + { + "collection_id": int, + "id": str, + "embedding": bytes, + "content": str, + "metadata": str, + }, + pk=("collection_id", "id"), + ) + + +@embeddings_migrations() +def m002_foreign_key(db): + db["embeddings"].add_foreign_key("collection_id", "collections", "id") + + +@embeddings_migrations() +def m003_add_updated(db): + db["embeddings"].add_column("updated", int) + # Pretty-print the schema + db["embeddings"].transform() + # Assume anything existing was last updated right now + db.query( + "update embeddings set updated = ? where updated is null", [int(time.time())] + ) + + +@embeddings_migrations() +def m004_store_content_hash(db): + db["embeddings"].add_column("content_hash", bytes) + db["embeddings"].transform( + column_order=( + "collection_id", + "id", + "embedding", + "content", + "content_hash", + "metadata", + "updated", + ) + ) + + # Register functions manually so we can de-register later + def md5(text): + return hashlib.md5(text.encode("utf8")).digest() + + def random_md5(): + return hashlib.md5(str(time.time()).encode("utf8")).digest() + + db.conn.create_function("temp_md5", 1, md5) + db.conn.create_function("temp_random_md5", 0, random_md5) + + with db.conn: + db.execute(""" + update embeddings + set content_hash = temp_md5(content) + where content is not null + """) + db.execute(""" + update embeddings + set content_hash = temp_random_md5() + where content is null + """) + + db["embeddings"].create_index(["content_hash"]) + + # De-register functions + db.conn.create_function("temp_md5", 1, None) + db.conn.create_function("temp_random_md5", 0, None) + + +@embeddings_migrations() +def m005_add_content_blob(db): + db["embeddings"].add_column("content_blob", bytes) + db["embeddings"].transform( + column_order=("collection_id", "id", "embedding", "content", "content_blob") + ) diff --git a/build/lib/llm/errors.py b/build/lib/llm/errors.py new file mode 100644 index 000000000..10f50bb5a --- /dev/null +++ b/build/lib/llm/errors.py @@ -0,0 +1,6 @@ +class ModelError(Exception): + "Models can raise this error, which will be displayed to the user" + + +class NeedsKeyException(ModelError): + "Model needs an API key which has not been provided" diff --git a/build/lib/llm/hookspecs.py b/build/lib/llm/hookspecs.py new file mode 100644 index 000000000..7ab555199 --- /dev/null +++ b/build/lib/llm/hookspecs.py @@ -0,0 +1,35 @@ +from pluggy import HookimplMarker +from pluggy import HookspecMarker + +hookspec = HookspecMarker("llm") +hookimpl = HookimplMarker("llm") + + +@hookspec +def register_commands(cli): + """Register additional CLI commands, e.g. 'llm mycommand ...'""" + + +@hookspec +def register_models(register, model_aliases): + "Register additional model instances representing LLM models that can be called" + + +@hookspec +def register_embedding_models(register): + "Register additional model instances that can be used for embedding" + + +@hookspec +def register_template_loaders(register): + "Register additional template loaders with prefixes" + + +@hookspec +def register_fragment_loaders(register): + "Register additional fragment loaders with prefixes" + + +@hookspec +def register_tools(register): + "Register functions that can be used as tools by the LLMs" diff --git a/build/lib/llm/migrations.py b/build/lib/llm/migrations.py new file mode 100644 index 000000000..f2ca04651 --- /dev/null +++ b/build/lib/llm/migrations.py @@ -0,0 +1,420 @@ +import datetime +from typing import Callable, List + +MIGRATIONS: List[Callable] = [] +migration = MIGRATIONS.append + + +def migrate(db): + ensure_migrations_table(db) + already_applied = {r["name"] for r in db["_llm_migrations"].rows} + for fn in MIGRATIONS: + name = fn.__name__ + if name not in already_applied: + fn(db) + db["_llm_migrations"].insert( + { + "name": name, + "applied_at": str(datetime.datetime.now(datetime.timezone.utc)), + } + ) + already_applied.add(name) + + +def ensure_migrations_table(db): + if not db["_llm_migrations"].exists(): + db["_llm_migrations"].create( + { + "name": str, + "applied_at": str, + }, + pk="name", + ) + + +@migration +def m001_initial(db): + # Ensure the original table design exists, so other migrations can run + if db["log"].exists(): + # It needs to have the chat_id column + if "chat_id" not in db["log"].columns_dict: + db["log"].add_column("chat_id") + return + db["log"].create( + { + "provider": str, + "system": str, + "prompt": str, + "chat_id": str, + "response": str, + "model": str, + "timestamp": str, + } + ) + + +@migration +def m002_id_primary_key(db): + db["log"].transform(pk="id") + + +@migration +def m003_chat_id_foreign_key(db): + db["log"].transform(types={"chat_id": int}) + db["log"].add_foreign_key("chat_id", "log", "id") + + +@migration +def m004_column_order(db): + db["log"].transform( + column_order=( + "id", + "model", + "timestamp", + "prompt", + "system", + "response", + "chat_id", + ) + ) + + +@migration +def m004_drop_provider(db): + db["log"].transform(drop=("provider",)) + + +@migration +def m005_debug(db): + db["log"].add_column("debug", str) + db["log"].add_column("duration_ms", int) + + +@migration +def m006_new_logs_table(db): + columns = db["log"].columns_dict + for column, type in ( + ("options_json", str), + ("prompt_json", str), + ("response_json", str), + ("reply_to_id", int), + ): + # It's possible people running development code like myself + # might have accidentally created these columns already + if column not in columns: + db["log"].add_column(column, type) + + # Use .transform() to rename options and timestamp_utc, and set new order + db["log"].transform( + column_order=( + "id", + "model", + "prompt", + "system", + "prompt_json", + "options_json", + "response", + "response_json", + "reply_to_id", + "chat_id", + "duration_ms", + "timestamp_utc", + ), + rename={ + "timestamp": "timestamp_utc", + "options": "options_json", + }, + ) + + +@migration +def m007_finish_logs_table(db): + db["log"].transform( + drop={"debug"}, + rename={"timestamp_utc": "datetime_utc"}, + drop_foreign_keys=("chat_id",), + ) + with db.conn: + db.execute("alter table log rename to logs") + + +@migration +def m008_reply_to_id_foreign_key(db): + db["logs"].add_foreign_key("reply_to_id", "logs", "id") + + +@migration +def m008_fix_column_order_in_logs(db): + # reply_to_id ended up at the end after foreign key added + db["logs"].transform( + column_order=( + "id", + "model", + "prompt", + "system", + "prompt_json", + "options_json", + "response", + "response_json", + "reply_to_id", + "chat_id", + "duration_ms", + "timestamp_utc", + ), + ) + + +@migration +def m009_delete_logs_table_if_empty(db): + # We moved to a new table design, but we don't delete the table + # if someone has put data in it + if not db["logs"].count: + db["logs"].drop() + + +@migration +def m010_create_new_log_tables(db): + db["conversations"].create( + { + "id": str, + "name": str, + "model": str, + }, + pk="id", + ) + db["responses"].create( + { + "id": str, + "model": str, + "prompt": str, + "system": str, + "prompt_json": str, + "options_json": str, + "response": str, + "response_json": str, + "conversation_id": str, + "duration_ms": int, + "datetime_utc": str, + }, + pk="id", + foreign_keys=(("conversation_id", "conversations", "id"),), + ) + + +@migration +def m011_fts_for_responses(db): + db["responses"].enable_fts(["prompt", "response"], create_triggers=True) + + +@migration +def m012_attachments_tables(db): + db["attachments"].create( + { + "id": str, + "type": str, + "path": str, + "url": str, + "content": bytes, + }, + pk="id", + ) + db["prompt_attachments"].create( + { + "response_id": str, + "attachment_id": str, + "order": int, + }, + foreign_keys=( + ("response_id", "responses", "id"), + ("attachment_id", "attachments", "id"), + ), + pk=("response_id", "attachment_id"), + ) + + +@migration +def m013_usage(db): + db["responses"].add_column("input_tokens", int) + db["responses"].add_column("output_tokens", int) + db["responses"].add_column("token_details", str) + + +@migration +def m014_schemas(db): + db["schemas"].create( + { + "id": str, + "content": str, + }, + pk="id", + ) + db["responses"].add_column("schema_id", str, fk="schemas", fk_col="id") + # Clean up SQL create table indentation + db["responses"].transform() + # These changes may have dropped the FTS configuration, fix that + db["responses"].enable_fts( + ["prompt", "response"], create_triggers=True, replace=True + ) + + +@migration +def m015_fragments_tables(db): + db["fragments"].create( + { + "id": int, + "hash": str, + "content": str, + "datetime_utc": str, + "source": str, + }, + pk="id", + ) + db["fragments"].create_index(["hash"], unique=True) + db["fragment_aliases"].create( + { + "alias": str, + "fragment_id": int, + }, + foreign_keys=(("fragment_id", "fragments", "id"),), + pk="alias", + ) + db["prompt_fragments"].create( + { + "response_id": str, + "fragment_id": int, + "order": int, + }, + foreign_keys=( + ("response_id", "responses", "id"), + ("fragment_id", "fragments", "id"), + ), + pk=("response_id", "fragment_id"), + ) + db["system_fragments"].create( + { + "response_id": str, + "fragment_id": int, + "order": int, + }, + foreign_keys=( + ("response_id", "responses", "id"), + ("fragment_id", "fragments", "id"), + ), + pk=("response_id", "fragment_id"), + ) + + +@migration +def m016_fragments_table_pks(db): + # The same fragment can be attached to a response multiple times + # https://github.com/simonw/llm/issues/863#issuecomment-2781720064 + db["prompt_fragments"].transform(pk=("response_id", "fragment_id", "order")) + db["system_fragments"].transform(pk=("response_id", "fragment_id", "order")) + + +@migration +def m017_tools_tables(db): + db["tools"].create( + { + "id": int, + "hash": str, + "name": str, + "description": str, + "input_schema": str, + }, + pk="id", + ) + db["tools"].create_index(["hash"], unique=True) + # Many-to-many relationship between tools and responses + db["tool_responses"].create( + { + "tool_id": int, + "response_id": str, + }, + foreign_keys=( + ("tool_id", "tools", "id"), + ("response_id", "responses", "id"), + ), + pk=("tool_id", "response_id"), + ) + # tool_calls and tool_results are one-to-many against responses + db["tool_calls"].create( + { + "id": int, + "response_id": str, + "tool_id": int, + "name": str, + "arguments": str, + "tool_call_id": str, + }, + pk="id", + foreign_keys=( + ("response_id", "responses", "id"), + ("tool_id", "tools", "id"), + ), + ) + db["tool_results"].create( + { + "id": int, + "response_id": str, + "tool_id": int, + "name": str, + "output": str, + "tool_call_id": str, + }, + pk="id", + foreign_keys=( + ("response_id", "responses", "id"), + ("tool_id", "tools", "id"), + ), + ) + + +@migration +def m017_tools_plugin(db): + db["tools"].add_column("plugin") + + +@migration +def m018_tool_instances(db): + # Used to track instances of Toolbox classes that may be + # used multiple times by different tools + db["tool_instances"].create( + { + "id": int, + "plugin": str, + "name": str, + "arguments": str, + }, + pk="id", + ) + # We record which instance was used only on the results + db["tool_results"].add_column("instance_id", fk="tool_instances") + + +@migration +def m019_resolved_model(db): + # For models like gemini-1.5-flash-latest where we wish to record + # the resolved model name in addition to the alias + db["responses"].add_column("resolved_model", str) + + +@migration +def m020_tool_results_attachments(db): + db["tool_results_attachments"].create( + { + "tool_result_id": int, + "attachment_id": str, + "order": int, + }, + foreign_keys=( + ("tool_result_id", "tool_results", "id"), + ("attachment_id", "attachments", "id"), + ), + pk=("tool_result_id", "attachment_id"), + ) + + +@migration +def m021_tool_results_exception(db): + db["tool_results"].add_column("exception", str) diff --git a/build/lib/llm/models.py b/build/lib/llm/models.py new file mode 100644 index 000000000..8600eb402 --- /dev/null +++ b/build/lib/llm/models.py @@ -0,0 +1,2966 @@ +import asyncio +import base64 +from condense_json import condense_json +from dataclasses import dataclass, field +import datetime +from .errors import NeedsKeyException +import hashlib +import httpx +from itertools import islice +from pathlib import Path +import re +import time +from types import MethodType +from typing import ( + Any, + AsyncGenerator, + AsyncIterator, + Awaitable, + Callable, + Dict, + Iterable, + Iterator, + List, + Optional, + Set, + Union, + get_type_hints, +) +from .serialization import ResponseDict +from .utils import ( + ensure_fragment, + ensure_tool, + make_schema_id, + mimetype_from_path, + mimetype_from_string, + token_usage_string, + monotonic_ulid, + Fragment, +) +from abc import ABC, abstractmethod +import inspect +import json +from pydantic import BaseModel, ConfigDict, create_model + +CONVERSATION_NAME_LENGTH = 32 + + +@dataclass +class Usage: + "Token usage information from a model response." + + input: Optional[int] = None + output: Optional[int] = None + details: Optional[Dict[str, Any]] = None + + +@dataclass +class Attachment: + "An attachment (image, audio, etc) to include with a prompt." + + type: Optional[str] = None + path: Optional[str] = None + url: Optional[str] = None + content: Optional[bytes] = None + _id: Optional[str] = None + + def id(self): + # Hash of the binary content, or of '{"url": "https://..."}' for URL attachments + if self._id is None: + if self.content: + self._id = hashlib.sha256(self.content).hexdigest() + elif self.path: + self._id = hashlib.sha256(Path(self.path).read_bytes()).hexdigest() + else: + self._id = hashlib.sha256( + json.dumps({"url": self.url}).encode("utf-8") + ).hexdigest() + return self._id + + def resolve_type(self): + "Return the content type, guessing from content if not specified." + if self.type: + return self.type + # Derive it from path or url or content + if self.path: + return mimetype_from_path(self.path) + if self.url: + response = httpx.head(self.url) + response.raise_for_status() + return response.headers.get("content-type") + if self.content: + return mimetype_from_string(self.content) + raise ValueError("Attachment has no type and no content to derive it from") + + def content_bytes(self): + "Return the binary content, reading from path or URL if needed." + content = self.content + if not content: + if self.path: + content = Path(self.path).read_bytes() + elif self.url: + response = httpx.get(self.url) + response.raise_for_status() + content = response.content + return content + + def base64_content(self): + "Return the content as a base64-encoded string." + return base64.b64encode(self.content_bytes()).decode("utf-8") + + def __repr__(self): + info = [f"" + + @classmethod + def from_row(cls, row): + return cls( + _id=row["id"], + type=row["type"], + path=row["path"], + url=row["url"], + content=row["content"], + ) + + +@dataclass +class Tool: + "A tool that can be called by a model." + + name: str + description: Optional[str] = None + input_schema: Dict = field(default_factory=dict) + implementation: Optional[Callable] = None + plugin: Optional[str] = None # plugin tool came from, e.g. 'llm_tools_sqlite' + + def __post_init__(self): + # Convert Pydantic model to JSON schema if needed + self.input_schema = _ensure_dict_schema(self.input_schema) + + def hash(self): + """Hash for tool based on its name, description and input schema (preserving key order)""" + to_hash = { + "name": self.name, + "description": self.description, + "input_schema": self.input_schema, + } + if self.plugin: + to_hash["plugin"] = self.plugin + return hashlib.sha256(json.dumps(to_hash).encode("utf-8")).hexdigest() + + @classmethod + def function(cls, function, name=None, description=None): + """ + Turn a Python function into a Tool object by: + - Extracting the function name + - Using the function docstring for the Tool description + - Building a Pydantic model for inputs by inspecting the function signature + - Building a Pydantic model for the return value by using the function's return annotation + """ + if not name and function.__name__ == "": + raise ValueError( + "Cannot create a Tool from a lambda function without providing name=" + ) + + return cls( + name=name or function.__name__, + description=description or function.__doc__ or None, + input_schema=_get_arguments_input_schema(function, name), + implementation=function, + ) + + +def _get_arguments_input_schema(function, name): + signature = inspect.signature(function) + type_hints = get_type_hints(function) + fields = {} + for param_name, param in signature.parameters.items(): + if param_name == "self": + continue + # Determine the type annotation (default to string if missing) + annotated_type = type_hints.get(param_name, str) + + # Handle default value if present; if there's no default, use '...' + if param.default is inspect.Parameter.empty: + fields[param_name] = (annotated_type, ...) + else: + fields[param_name] = (annotated_type, param.default) + + return create_model(f"{name}InputSchema", **fields) + + +class Toolbox: + name: Optional[str] = None + instance_id: Optional[int] = None + _blocked = ( + "tools", + "add_tool", + "method_tools", + "__init_subclass__", + "prepare", + "prepare_async", + ) + _extra_tools: List[Tool] = [] + _config: Dict[str, Any] = {} + _prepared: bool = False + _async_prepared: bool = False + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + + original_init = cls.__init__ + + def wrapped_init(self, *args, **kwargs): + # Track args/kwargs passed to constructor in self._config + # so we can serialize them to a database entry later on + sig = inspect.signature(original_init) + bound = sig.bind(self, *args, **kwargs) + bound.apply_defaults() + + self._config = { + name: value + for name, value in bound.arguments.items() + if name != "self" + and sig.parameters[name].kind + not in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD) + } + self._extra_tools = [] + + original_init(self, *args, **kwargs) + + cls.__init__ = wrapped_init + + @classmethod + def method_tools(cls) -> List[Tool]: + tools = [] + for method_name in dir(cls): + if method_name.startswith("_") or method_name in cls._blocked: + continue + method = getattr(cls, method_name) + if callable(method): + tool = Tool.function( + method, + name="{}_{}".format(cls.__name__, method_name), + ) + tools.append(tool) + return tools + + def tools(self) -> Iterable[Tool]: + "Returns an llm.Tool() for each class method, plus any extras registered with add_tool()" + # method_tools() returns unbound methods, we need bound methods here: + for name in dir(self): + if name.startswith("_") or name in self._blocked: + continue + attr = getattr(self, name) + if callable(attr): + tool = Tool.function(attr, name=f"{self.__class__.__name__}_{name}") + tool.plugin = getattr(self, "plugin", None) + yield tool + yield from self._extra_tools + + def add_tool( + self, tool_or_function: Union[Tool, Callable[..., Any]], pass_self: bool = False + ): + "Add a tool to this toolbox" + + def _upgrade(fn): + if pass_self: + return MethodType(fn, self) + return fn + + if isinstance(tool_or_function, Tool): + self._extra_tools.append(tool_or_function) + elif callable(tool_or_function): + self._extra_tools.append(Tool.function(_upgrade(tool_or_function))) + else: + raise ValueError("Tool must be an instance of Tool or a callable function") + + def prepare(self): + """ + Over-ride this to perform setup (and .add_tool() calls) before the toolbox is used. + Implement a similar prepare_async() method for async setup. + """ + pass + + async def prepare_async(self): + """ + Over-ride this to perform async setup (and .add_tool() calls) before the toolbox is used. + """ + pass + + +@dataclass +class ToolCall: + "A request by the model to call a tool." + + name: str + arguments: dict + tool_call_id: Optional[str] = None + + +@dataclass +class ToolResult: + "The result of executing a tool call." + + name: str + output: str + attachments: List[Attachment] = field(default_factory=list) + tool_call_id: Optional[str] = None + instance: Optional[Toolbox] = None + exception: Optional[Exception] = None + + +@dataclass +class ToolOutput: + "Tool functions can return output with extra attachments" + + output: Optional[Union[str, dict, list, bool, int, float]] = None + attachments: List[Attachment] = field(default_factory=list) + + +ToolDef = Union[Tool, Toolbox, Callable[..., Any]] +BeforeCallSync = Callable[[Optional[Tool], ToolCall], None] +AfterCallSync = Callable[[Tool, ToolCall, ToolResult], None] +BeforeCallAsync = Callable[[Optional[Tool], ToolCall], Union[None, Awaitable[None]]] +AfterCallAsync = Callable[[Tool, ToolCall, ToolResult], Union[None, Awaitable[None]]] + + +class CancelToolCall(Exception): + pass + + +@dataclass +class Prompt: + "The prompt being sent to the model." + + _prompt: Optional[str] + model: "Model" + fragments: Optional[List[Union[str, Fragment]]] + attachments: Optional[List[Attachment]] + _system: Optional[str] + system_fragments: Optional[List[Union[str, Fragment]]] + prompt_json: Optional[str] + schema: Optional[Union[Dict, type[BaseModel]]] + tools: List[Tool] + tool_results: List[ToolResult] + options: "Options" + + def __init__( + self, + prompt, + model, + *, + fragments=None, + attachments=None, + system=None, + system_fragments=None, + prompt_json=None, + options=None, + schema=None, + tools=None, + tool_results=None, + messages=None, + ): + self._prompt = prompt + self.model = model + self.attachments = list(attachments or []) + self.fragments = fragments or [] + self._system = system + self.system_fragments = system_fragments or [] + self.prompt_json = prompt_json + if schema and not isinstance(schema, dict) and issubclass(schema, BaseModel): + schema = schema.model_json_schema() + self.schema = schema + self.tools = _wrap_tools(tools or []) + self.tool_results = tool_results or [] + self.options = options or {} + # Explicit messages= list, if the caller supplied one. Copied so + # later mutation by the caller doesn't alter the Prompt. + self._explicit_messages = list(messages) if messages is not None else None + + @property + def prompt(self): + "The text of the prompt, with any fragments concatenated." + return "\n".join(self.fragments + ([self._prompt] if self._prompt else [])) + + @property + def system(self): + "The system prompt, with any system fragments concatenated." + bits = [ + bit.strip() + for bit in (self.system_fragments + [self._system or ""]) + if bit.strip() + ] + return "\n\n".join(bits) + + @property + def messages(self): + """Canonical list of Message objects for this prompt. + + **Invariant:** this property returns exactly what the model + was (or will be) sent for this turn — the full chain including + any prior conversation history. + + - If ``messages=`` was passed explicitly, it is authoritative: + returned verbatim. Other kwargs (``prompt=``, ``system=``, + ``attachments=``, ``tool_results=``) are ignored for the + messages list (they remain available via ``prompt.prompt``, + ``prompt.system``, etc., for adapters that still read them). + - Otherwise the list is synthesized from the legacy kwargs + (system, tool_results, prompt, attachments), producing just + the current turn — prior history is not folded in, because + no conversation context is reachable here. + + Conversation.prompt / AsyncConversation.prompt / reply() all + pre-compute the full chain and pass it as ``messages=``, so + ``response.prompt.messages`` after those paths is the full + chain. + """ + from .parts import ( + AttachmentPart, + Message, + TextPart, + ToolResultPart, + ) + + if self._explicit_messages is not None: + return list(self._explicit_messages) + + result: List["Message"] = [] + + if self.system: + result.append( + Message(role="system", parts=[TextPart(text=self.system)]) + ) + + if self.tool_results: + result.append( + Message( + role="tool", + parts=[ + ToolResultPart( + name=tr.name, + output=tr.output, + tool_call_id=tr.tool_call_id, + ) + for tr in self.tool_results + ], + ) + ) + + user_parts: List[Any] = [] + if self.prompt: + user_parts.append(TextPart(text=self.prompt)) + for att in self.attachments: + user_parts.append(AttachmentPart(attachment=att)) + if user_parts: + result.append(Message(role="user", parts=user_parts)) + + return result + + +def _wrap_tools(tools: List[ToolDef]) -> List[Tool]: + wrapped_tools = [] + for tool in tools: + if isinstance(tool, Tool): + wrapped_tools.append(tool) + elif isinstance(tool, Toolbox): + wrapped_tools.extend(tool.tools()) + elif callable(tool): + wrapped_tools.append(Tool.function(tool)) + else: + raise ValueError(f"Invalid tool: {tool}") + return wrapped_tools + + +@dataclass +class _BaseConversation: + model: "_BaseModel" + id: str = field(default_factory=lambda: str(monotonic_ulid()).lower()) + name: Optional[str] = None + responses: List["_BaseResponse"] = field(default_factory=list) + tools: Optional[List[ToolDef]] = None + chain_limit: Optional[int] = None + + @classmethod + @abstractmethod + def from_row(cls, row: Any) -> "_BaseConversation": + raise NotImplementedError + + def _build_full_chain( + self, + prompt: Optional[str], + attachments, + tool_results, + explicit_messages, + ) -> List[Any]: + """Build the full message chain for the next turn. + + Walks this conversation's responses to collect prior history, + then appends the new turn's content (explicit messages first, + or synthesized from prompt/attachments/tool_results). + + Returns the list that should be passed as ``messages=`` to the + Prompt constructor so that ``response.prompt.messages`` equals + exactly what the model sees. + + If ``explicit_messages`` is provided, the caller has opted out + of history walking — the list is used as-is. + """ + from .parts import ( + AttachmentPart, + Message, + TextPart, + ToolResultPart, + ) + + if explicit_messages is not None: + return list(explicit_messages) + + chain: List[Any] = [] + for prev in self.responses: + # prev.prompt.messages already contains prev's full input + # chain under the new invariant, but for the FIRST hop into + # a conversation we defensively de-duplicate by only + # concatenating the last response's full chain (which + # transitively includes everything before it). + pass + if self.responses: + last = self.responses[-1] + chain.extend(last.prompt.messages) + # Append that response's own output (structured messages). + try: + chain.extend(last.messages) + except ValueError: + # AsyncResponse not yet awaited — the caller shouldn't + # be constructing a next turn without awaiting first. + pass + + # Append the new turn's input + if tool_results: + chain.append( + Message( + role="tool", + parts=[ + ToolResultPart( + name=tr.name, + output=tr.output, + tool_call_id=tr.tool_call_id, + ) + for tr in tool_results + ], + ) + ) + + user_parts: List[Any] = [] + if prompt: + user_parts.append(TextPart(text=prompt)) + for att in attachments or []: + user_parts.append(AttachmentPart(attachment=att)) + if user_parts: + chain.append(Message(role="user", parts=user_parts)) + + return chain + + +@dataclass +class Conversation(_BaseConversation): + before_call: Optional[BeforeCallSync] = None + after_call: Optional[AfterCallSync] = None + + def prompt( + self, + prompt: Optional[str] = None, + *, + fragments: Optional[List[Union[str, Fragment]]] = None, + attachments: Optional[List[Attachment]] = None, + system: Optional[str] = None, + schema: Optional[Union[dict, type[BaseModel]]] = None, + tools: Optional[List[ToolDef]] = None, + tool_results: Optional[List[ToolResult]] = None, + system_fragments: Optional[List[Union[str, Fragment]]] = None, + messages: Optional[List[Any]] = None, + stream: bool = True, + key: Optional[str] = None, + **options, + ) -> "Response": + # Build the authoritative chain so response.prompt.messages + # equals exactly what the model sees for this turn. + chain = self._build_full_chain( + prompt=prompt, + attachments=attachments, + tool_results=tool_results, + explicit_messages=messages, + ) + return Response( + Prompt( + prompt, + model=self.model, + fragments=fragments, + attachments=attachments, + system=system, + schema=schema, + tools=tools or self.tools, + tool_results=tool_results, + system_fragments=system_fragments, + messages=chain, + options=self.model.Options(**options), + ), + self.model, + stream, + conversation=self, + key=key, + ) + + def chain( + self, + prompt: Optional[str] = None, + *, + fragments: Optional[List[str]] = None, + attachments: Optional[List[Attachment]] = None, + system: Optional[str] = None, + system_fragments: Optional[List[str]] = None, + messages: Optional[List[Any]] = None, + stream: bool = True, + schema: Optional[Union[dict, type[BaseModel]]] = None, + tools: Optional[List[ToolDef]] = None, + tool_results: Optional[List[ToolResult]] = None, + chain_limit: Optional[int] = None, + before_call: Optional[BeforeCallSync] = None, + after_call: Optional[AfterCallSync] = None, + key: Optional[str] = None, + options: Optional[dict] = None, + ) -> "ChainResponse": + self.model._validate_attachments(attachments) + # Parity with Conversation.prompt: pre-bake the full chain so + # response.prompt.messages is authoritative for the first turn + # of the chain loop. Subsequent tool-result turns extend the + # chain via _chain_for_tool_results. + chain_messages = self._build_full_chain( + prompt=prompt, + attachments=attachments, + tool_results=tool_results, + explicit_messages=messages, + ) + return ChainResponse( + Prompt( + prompt, + fragments=fragments, + attachments=attachments, + system=system, + schema=schema, + tools=tools or self.tools, + tool_results=tool_results, + system_fragments=system_fragments, + messages=chain_messages, + model=self.model, + options=self.model.Options(**(options or {})), + ), + model=self.model, + stream=stream, + conversation=self, + key=key, + before_call=before_call or self.before_call, + after_call=after_call or self.after_call, + chain_limit=chain_limit if chain_limit is not None else self.chain_limit, + ) + + @classmethod + def from_row(cls, row): + from llm import get_model + + return cls( + model=get_model(row["model"]), + id=row["id"], + name=row["name"], + ) + + def __repr__(self): + count = len(self.responses) + s = "s" if count == 1 else "" + return f"<{self.__class__.__name__}: {self.id} - {count} response{s}" + + +@dataclass +class AsyncConversation(_BaseConversation): + before_call: Optional[BeforeCallAsync] = None + after_call: Optional[AfterCallAsync] = None + + def chain( + self, + prompt: Optional[str] = None, + *, + fragments: Optional[List[str]] = None, + attachments: Optional[List[Attachment]] = None, + system: Optional[str] = None, + system_fragments: Optional[List[str]] = None, + messages: Optional[List[Any]] = None, + stream: bool = True, + schema: Optional[Union[dict, type[BaseModel]]] = None, + tools: Optional[List[ToolDef]] = None, + tool_results: Optional[List[ToolResult]] = None, + chain_limit: Optional[int] = None, + before_call: Optional[BeforeCallAsync] = None, + after_call: Optional[AfterCallAsync] = None, + key: Optional[str] = None, + options: Optional[dict] = None, + ) -> "AsyncChainResponse": + self.model._validate_attachments(attachments) + chain_messages = self._build_full_chain( + prompt=prompt, + attachments=attachments, + tool_results=tool_results, + explicit_messages=messages, + ) + return AsyncChainResponse( + Prompt( + prompt, + fragments=fragments, + attachments=attachments, + system=system, + schema=schema, + tools=tools or self.tools, + tool_results=tool_results, + system_fragments=system_fragments, + messages=chain_messages, + model=self.model, + options=self.model.Options(**(options or {})), + ), + model=self.model, + stream=stream, + conversation=self, + key=key, + before_call=before_call or self.before_call, + after_call=after_call or self.after_call, + chain_limit=chain_limit if chain_limit is not None else self.chain_limit, + ) + + def prompt( + self, + prompt: Optional[str] = None, + *, + fragments: Optional[List[str]] = None, + attachments: Optional[List[Attachment]] = None, + system: Optional[str] = None, + schema: Optional[Union[dict, type[BaseModel]]] = None, + tools: Optional[List[ToolDef]] = None, + tool_results: Optional[List[ToolResult]] = None, + system_fragments: Optional[List[str]] = None, + messages: Optional[List[Any]] = None, + stream: bool = True, + key: Optional[str] = None, + **options, + ) -> "AsyncResponse": + chain = self._build_full_chain( + prompt=prompt, + attachments=attachments, + tool_results=tool_results, + explicit_messages=messages, + ) + return AsyncResponse( + Prompt( + prompt, + model=self.model, + fragments=fragments, + attachments=attachments, + system=system, + schema=schema, + tools=tools, + tool_results=tool_results, + system_fragments=system_fragments, + messages=chain, + options=self.model.Options(**options), + ), + self.model, + stream, + conversation=self, + key=key, + ) + + def to_sync_conversation(self): + return Conversation( + model=self.model, + id=self.id, + name=self.name, + responses=[], # Because we only use this in logging + tools=self.tools, + chain_limit=self.chain_limit, + ) + + @classmethod + def from_row(cls, row): + from llm import get_async_model + + return cls( + model=get_async_model(row["model"]), + id=row["id"], + name=row["name"], + ) + + def __repr__(self): + count = len(self.responses) + s = "s" if count == 1 else "" + return f"<{self.__class__.__name__}: {self.id} - {count} response{s}" + + +FRAGMENT_SQL = """ +select + 'prompt' as fragment_type, + fragments.content, + pf."order" as ord +from prompt_fragments pf +join fragments on pf.fragment_id = fragments.id +where pf.response_id = :response_id +union all +select + 'system' as fragment_type, + fragments.content, + sf."order" as ord +from system_fragments sf +join fragments on sf.fragment_id = fragments.id +where sf.response_id = :response_id +order by fragment_type desc, ord asc; +""" + + +class _BaseResponse: + """Base response class shared between sync and async responses""" + + id: str + prompt: "Prompt" + stream: bool + resolved_model: Optional[str] = None + conversation: Optional["_BaseConversation"] = None + _key: Optional[str] = None + _tool_calls: List[ToolCall] = [] + + def __init__( + self, + prompt: Prompt, + model: "_BaseModel", + stream: bool, + conversation: Optional[_BaseConversation] = None, + key: Optional[str] = None, + ): + self.id = str(monotonic_ulid()).lower() + self.prompt = prompt + self._prompt_json = None + self.model = model + self.stream = stream + self._key = key + self._chunks: List[str] = [] + # Every StreamEvent ever yielded by execute(), in order. Plain + # str yields are wrapped as StreamEvent(type="text", part_index=0) + # so this buffer is the single source of truth for replay and + # for assembling response.messages. + self._stream_events: List[Any] = [] + # Plugins set this when the provider reports an opaque reasoning + # token count (no streamed reasoning text). _build_parts() + # prepends a ReasoningPart(redacted=True, token_count=N) when + # non-zero. + self._reasoning_token_count: int = 0 + self._done = False + self._tool_calls: List[ToolCall] = [] + self.response_json: Optional[Dict[str, Any]] = None + self.conversation = conversation + self.attachments: List[Attachment] = [] + self._start: Optional[float] = None + self._end: Optional[float] = None + self._start_utcnow: Optional[datetime.datetime] = None + self.input_tokens: Optional[int] = None + self.output_tokens: Optional[int] = None + self.token_details: Optional[dict] = None + self.done_callbacks: List[Callable] = [] + + if self.prompt.schema and not self.model.supports_schema: + raise ValueError(f"{self.model} does not support schemas") + + if self.prompt.tools and not self.model.supports_tools: + raise ValueError(f"{self.model} does not support tools") + + def _process_chunk(self, chunk): + """Normalize a chunk from execute() into a StreamEvent and return + the text str (or None) that __iter__ should yield. + + Plain str yields from legacy plugins are wrapped as text events + at part_index=0. Side effects: populates self._stream_events and + self._chunks. + """ + from .parts import StreamEvent + + if isinstance(chunk, StreamEvent): + self._stream_events.append(chunk) + if chunk.type == "text": + self._chunks.append(chunk.chunk) + return chunk.chunk + return None + # Legacy plain-str plugin. + event = StreamEvent(type="text", chunk=chunk, part_index=0) + self._stream_events.append(event) + self._chunks.append(chunk) + return chunk + + def _build_parts(self) -> List[Any]: + """Assemble Part objects from the accumulated stream events. + + Events sharing a part_index group into one Part. Mixing + families (text vs tool_call vs reasoning vs tool_result) at the + same index is a plugin bug — raises ValueError instead of + silently dropping content. + + Fallback: when no stream events were recorded (response was + rehydrated from SQLite via ``from_row``), synthesize a + TextPart from ``self._chunks`` plus any ``self._tool_calls`` + restored by the row loader. Reasoning signatures are not + recoverable from SQLite in this fallback — use + ``response.to_dict()`` / ``Response.from_dict()`` for + structure-preserving persistence. + """ + from .parts import ( + ReasoningPart, + TextPart, + ToolCallPart, + ToolResultPart, + ) + + if not self._stream_events: + # Rehydrated-from-SQLite path: assemble from _chunks + + # _tool_calls so response.messages isn't empty after + # from_row, and Conversation.prompt-built chains include + # the assistant turn on follow-up calls. + parts: List[Any] = [] + text = "".join(self._chunks) + if text: + parts.append(TextPart(text=text)) + for tc in self._tool_calls: + parts.append( + ToolCallPart( + name=tc.name, + arguments=tc.arguments or {}, + tool_call_id=tc.tool_call_id, + ) + ) + reasoning_token_count = getattr( + self, "_reasoning_token_count", 0 + ) + if reasoning_token_count: + parts.insert( + 0, + ReasoningPart( + text="", + redacted=True, + token_count=reasoning_token_count, + ), + ) + return parts + + def family(t: str) -> str: + if t in ("tool_call_name", "tool_call_args"): + return "tool_call" + return t + + parts: List[Any] = [] + current_index: Optional[int] = None + current_family: Optional[str] = None + text_buf: List[str] = [] + tool_name: Optional[str] = None + tool_args_buf: List[str] = [] + tool_call_id: Optional[str] = None + server_executed = False + tool_result_name: Optional[str] = None + pm_merged: Optional[Dict[str, Any]] = None + + def finalize(): + nonlocal pm_merged + if current_family is None: + return + if current_family == "text": + text = "".join(text_buf) + if text: + parts.append(TextPart(text=text, provider_metadata=pm_merged)) + elif current_family == "reasoning": + text = "".join(text_buf) + if text: + parts.append( + ReasoningPart(text=text, provider_metadata=pm_merged) + ) + elif current_family == "tool_call": + args_str = "".join(tool_args_buf) + try: + arguments = json.loads(args_str) if args_str else {} + except json.JSONDecodeError: + arguments = {"_raw": args_str} + parts.append( + ToolCallPart( + name=tool_name or "", + arguments=arguments, + tool_call_id=tool_call_id, + server_executed=server_executed, + provider_metadata=pm_merged, + ) + ) + elif current_family == "tool_result": + parts.append( + ToolResultPart( + name=tool_result_name or "", + output="".join(text_buf), + tool_call_id=tool_call_id, + server_executed=server_executed, + provider_metadata=pm_merged, + ) + ) + + for event in self._stream_events: + ev_family = family(event.type) + if event.part_index != current_index: + finalize() + current_index = event.part_index + current_family = ev_family + text_buf = [] + tool_name = None + tool_args_buf = [] + tool_call_id = None + server_executed = False + tool_result_name = None + pm_merged = None + elif current_family is not None and ev_family != current_family: + raise ValueError( + f"StreamEvent type {event.type!r} is incompatible with " + f"prior type at part_index={event.part_index}. " + "Allocate a new part_index for a different content type." + ) + + if event.type == "text": + text_buf.append(event.chunk) + elif event.type == "reasoning": + text_buf.append(event.chunk) + elif event.type == "tool_call_name": + tool_name = (tool_name or "") + event.chunk + if event.tool_call_id: + tool_call_id = event.tool_call_id + if event.server_executed: + server_executed = True + elif event.type == "tool_call_args": + tool_args_buf.append(event.chunk) + if event.tool_call_id and tool_call_id is None: + tool_call_id = event.tool_call_id + if event.server_executed: + server_executed = True + elif event.type == "tool_result": + text_buf.append(event.chunk) + if event.tool_call_id and tool_call_id is None: + tool_call_id = event.tool_call_id + if event.server_executed: + server_executed = True + if event.tool_name: + tool_result_name = event.tool_name + + if event.provider_metadata: + merged = dict(pm_merged) if pm_merged else {} + for k, v in event.provider_metadata.items(): + merged[k] = v + pm_merged = merged + + finalize() + + if self._reasoning_token_count: + parts.insert( + 0, + ReasoningPart( + text="", + redacted=True, + token_count=self._reasoning_token_count, + ), + ) + + return parts + + def add_tool_call(self, tool_call: ToolCall): + self._tool_calls.append(tool_call) + + def set_usage( + self, + *, + input: Optional[int] = None, + output: Optional[int] = None, + details: Optional[dict] = None, + ): + self.input_tokens = input + self.output_tokens = output + self.token_details = details + + def set_resolved_model(self, model_id: str): + self.resolved_model = model_id + + @classmethod + def from_row(cls, db, row, _async=False): + from llm import get_model, get_async_model + + if _async: + model = get_async_model(row["model"]) + else: + model = get_model(row["model"]) + + # Schema + schema = None + if row["schema_id"]: + schema = json.loads(db["schemas"].get(row["schema_id"])["content"]) + + # Tool definitions and results for prompt + tools = [ + Tool( + name=tool_row["name"], + description=tool_row["description"], + input_schema=json.loads(tool_row["input_schema"]), + # In this case we don't have a reference to the actual Python code + # but that's OK, we should not need it for prompts deserialized from DB + implementation=None, + plugin=tool_row["plugin"], + ) + for tool_row in db.query( + """ + select tools.* from tools + join tool_responses on tools.id = tool_responses.tool_id + where tool_responses.response_id = ? + """, + [row["id"]], + ) + ] + tool_results = [ + ToolResult( + name=tool_results_row["name"], + output=tool_results_row["output"], + tool_call_id=tool_results_row["tool_call_id"], + ) + for tool_results_row in db.query( + """ + select * from tool_results + where response_id = ? + """, + [row["id"]], + ) + ] + + all_fragments = list(db.query(FRAGMENT_SQL, {"response_id": row["id"]})) + fragments = [ + row["content"] for row in all_fragments if row["fragment_type"] == "prompt" + ] + system_fragments = [ + row["content"] for row in all_fragments if row["fragment_type"] == "system" + ] + response = cls( + model=model, + prompt=Prompt( + prompt=row["prompt"], + model=model, + fragments=fragments, + attachments=[], + system=row["system"], + schema=schema, + tools=tools, + tool_results=tool_results, + system_fragments=system_fragments, + options=model.Options(**json.loads(row["options_json"])), + ), + stream=False, + ) + prompt_json = json.loads(row["prompt_json"] or "null") + response.id = row["id"] + response._prompt_json = prompt_json + response.response_json = json.loads(row["response_json"] or "null") + response._done = True + response._chunks = [row["response"]] + # Attachments + response.attachments = [ + Attachment.from_row(attachment_row) + for attachment_row in db.query( + """ + select attachments.* from attachments + join prompt_attachments on attachments.id = prompt_attachments.attachment_id + where prompt_attachments.response_id = ? + order by prompt_attachments."order" + """, + [row["id"]], + ) + ] + # Tool calls + response._tool_calls = [ + ToolCall( + name=tool_row["name"], + arguments=json.loads(tool_row["arguments"]), + tool_call_id=tool_row["tool_call_id"], + ) + for tool_row in db.query( + """ + select * from tool_calls + where response_id = ? + order by tool_call_id + """, + [row["id"]], + ) + ] + + return response + + def token_usage(self) -> str: + return token_usage_string( + self.input_tokens, self.output_tokens, self.token_details + ) + + def log_to_db(self, db): + conversation = self.conversation + if not conversation: + conversation = Conversation(model=self.model) + db["conversations"].insert( + { + "id": conversation.id, + "name": _conversation_name( + self.prompt.prompt or self.prompt.system or "" + ), + "model": conversation.model.model_id, + }, + ignore=True, + ) + schema_id = None + if self.prompt.schema: + schema_id, schema_json = make_schema_id(self.prompt.schema) + db["schemas"].insert({"id": schema_id, "content": schema_json}, ignore=True) + + response_id = self.id + replacements = {} + # Include replacements from previous responses + for previous_response in conversation.responses[:-1]: + for fragment in (previous_response.prompt.fragments or []) + ( + previous_response.prompt.system_fragments or [] + ): + fragment_id = ensure_fragment(db, fragment) + replacements[f"f:{fragment_id}"] = fragment + replacements[f"r:{previous_response.id}"] = ( + previous_response.text_or_raise() + ) + + for i, fragment in enumerate(self.prompt.fragments): + fragment_id = ensure_fragment(db, fragment) + replacements[f"f{fragment_id}"] = fragment + db["prompt_fragments"].insert( + { + "response_id": response_id, + "fragment_id": fragment_id, + "order": i, + }, + ) + for i, fragment in enumerate(self.prompt.system_fragments): + fragment_id = ensure_fragment(db, fragment) + replacements[f"f{fragment_id}"] = fragment + db["system_fragments"].insert( + { + "response_id": response_id, + "fragment_id": fragment_id, + "order": i, + }, + ) + + response_text = self.text_or_raise() + replacements[f"r:{response_id}"] = response_text + json_data = self.json() + + response = { + "id": response_id, + "model": self.model.model_id, + "prompt": self.prompt._prompt, + "system": self.prompt._system, + "prompt_json": condense_json(self._prompt_json, replacements), + "options_json": { + key: value + for key, value in dict(self.prompt.options).items() + if value is not None + }, + "response": response_text, + "response_json": condense_json(json_data, replacements), + "conversation_id": conversation.id, + "duration_ms": self.duration_ms(), + "datetime_utc": self.datetime_utc(), + "input_tokens": self.input_tokens, + "output_tokens": self.output_tokens, + "token_details": ( + json.dumps(self.token_details) if self.token_details else None + ), + "schema_id": schema_id, + "resolved_model": self.resolved_model, + } + db["responses"].insert(response) + + # Persist any attachments - loop through with index + for index, attachment in enumerate(self.prompt.attachments): + attachment_id = attachment.id() + db["attachments"].insert( + { + "id": attachment_id, + "type": attachment.resolve_type(), + "path": attachment.path, + "url": attachment.url, + "content": attachment.content, + }, + replace=True, + ) + db["prompt_attachments"].insert( + { + "response_id": response_id, + "attachment_id": attachment_id, + "order": index, + }, + ) + + # Persist any tools, tool calls and tool results + tool_ids_by_name = {} + for tool in self.prompt.tools: + tool_id = ensure_tool(db, tool) + tool_ids_by_name[tool.name] = tool_id + db["tool_responses"].insert( + { + "tool_id": tool_id, + "response_id": response_id, + } + ) + for tool_call in self.tool_calls(): # TODO Should be _or_raise() + db["tool_calls"].insert( + { + "response_id": response_id, + "tool_id": tool_ids_by_name.get(tool_call.name) or None, + "name": tool_call.name, + "arguments": json.dumps(tool_call.arguments), + "tool_call_id": tool_call.tool_call_id, + } + ) + for tool_result in self.prompt.tool_results: + instance_id = None + if tool_result.instance: + try: + if not tool_result.instance.instance_id: + tool_result.instance.instance_id = ( + db["tool_instances"] + .insert( + { + "plugin": tool.plugin, + "name": tool.name.split("_")[0], + "arguments": json.dumps( + tool_result.instance._config + ), + } + ) + .last_pk + ) + instance_id = tool_result.instance.instance_id + except AttributeError: + pass + tool_result_id = ( + db["tool_results"] + .insert( + { + "response_id": response_id, + "tool_id": tool_ids_by_name.get(tool_result.name) or None, + "name": tool_result.name, + "output": tool_result.output, + "tool_call_id": tool_result.tool_call_id, + "instance_id": instance_id, + "exception": ( + ( + "{}: {}".format( + tool_result.exception.__class__.__name__, + str(tool_result.exception), + ) + ) + if tool_result.exception + else None + ), + } + ) + .last_pk + ) + # Persist attachments for tool results + for index, attachment in enumerate(tool_result.attachments): + attachment_id = attachment.id() + db["attachments"].insert( + { + "id": attachment_id, + "type": attachment.resolve_type(), + "path": attachment.path, + "url": attachment.url, + "content": attachment.content, + }, + replace=True, + ) + db["tool_results_attachments"].insert( + { + "tool_result_id": tool_result_id, + "attachment_id": attachment_id, + "order": index, + }, + ) + + +def _response_to_dict(response: "_BaseResponse") -> ResponseDict: + """Shared serializer for Response.to_dict / AsyncResponse.to_dict. + + The output is a JSON-safe dict — store it anywhere (file, Redis, + Postgres, HTTP body) and round-trip via Response.from_dict. + """ + options = { + key: value + for key, value in dict(response.prompt.options).items() + if value is not None + } + payload: Dict[str, Any] = { + "model": response.model.model_id, + "prompt": { + "messages": [m.to_dict() for m in response.prompt.messages], + }, + "messages": [m.to_dict() for m in response.messages], + } + if options: + payload["prompt"]["options"] = options + if response.prompt._system: + payload["prompt"]["system"] = response.prompt._system + # Optional audit fields — helpful for debugging, not needed for reply(). + if response.id: + payload["id"] = response.id + if response._done: + if response.input_tokens is not None or response.output_tokens is not None: + usage: Dict[str, Any] = {} + if response.input_tokens is not None: + usage["input"] = response.input_tokens + if response.output_tokens is not None: + usage["output"] = response.output_tokens + if response.token_details is not None: + usage["details"] = response.token_details + payload["usage"] = usage + if response._start_utcnow is not None: + payload["datetime_utc"] = response._start_utcnow.isoformat() + return payload + + +def _response_from_dict( + data: Dict[str, Any], + cls, + *, + model=None, + async_: bool = False, +) -> "_BaseResponse": + """Shared deserializer for Response.from_dict / AsyncResponse.from_dict.""" + from .parts import Message + + if model is None: + from llm import get_async_model, get_model + + getter = get_async_model if async_ else get_model + model = getter(data["model"]) + + prompt_data = data.get("prompt", {}) + input_messages = [ + Message.from_dict(m) for m in prompt_data.get("messages", []) + ] + output_messages = [ + Message.from_dict(m) for m in data.get("messages", []) + ] + + options_kwargs = prompt_data.get("options") or {} + system = prompt_data.get("system") + + prompt = Prompt( + None, + model=model, + messages=input_messages, + system=system, + options=model.Options(**options_kwargs), + ) + response = cls(prompt, model=model, stream=False) + # Preserve id for audit continuity. + if "id" in data: + response.id = data["id"] + # Rebuild chunks from the assistant's text parts so response.text() + # works without re-running the assembler. + from .parts import TextPart + + response._chunks = [ + p.text + for m in output_messages + for p in m.parts + if isinstance(p, TextPart) and p.text + ] + # Stash the structured output so response.messages returns the + # full picture (reasoning, tool calls, signatures) without needing + # a StreamEvent replay. + response._loaded_messages = output_messages + response._done = True + # Restore usage if present. + usage = data.get("usage") + if usage: + response.input_tokens = usage.get("input") + response.output_tokens = usage.get("output") + response.token_details = usage.get("details") + return response + + +class Response(_BaseResponse): + "Sync response from a model." + + model: "Model" + conversation: Optional["Conversation"] = None + + def reply( + self, + prompt: Optional[str] = None, + *, + messages: Optional[List[Any]] = None, + **kwargs, + ) -> "Response": + """Continue the conversation from this response. + + Builds the next turn's chain as + ``self.prompt.messages + self.messages + [user(prompt)]`` and + calls ``self.model.prompt(messages=chain, ...)``. No + Conversation object required — the Response carries everything + needed. + + If ``messages=`` is passed, its contents are appended to the + chain instead of (or in addition to) the ``prompt`` string. + """ + from .parts import Message, TextPart + + self._force() + chain: List[Any] = list(self.prompt.messages) + list(self.messages) + if prompt: + chain.append( + Message(role="user", parts=[TextPart(text=prompt)]) + ) + if messages: + chain.extend(messages) + return self.model.prompt(messages=chain, **kwargs) + + def to_dict(self) -> ResponseDict: + """Serialize this response for JSON persistence. + + Captures exactly what is needed to continue the conversation: + model id, the input chain that was sent + (``response.prompt.messages``), the structured assistant output + (``response.messages``), and any explicit options. Pair with + :meth:`Response.from_dict` to rehydrate and + :meth:`Response.reply` to continue. + + Returns :class:`~llm.serialization.ResponseDict`. + """ + return _response_to_dict(self) + + @classmethod + def from_dict( + cls, + data: ResponseDict, + *, + model: Optional["Model"] = None, + ) -> "Response": + """Rehydrate a Response from a ``to_dict()`` payload. + + The returned Response is in the ``_done`` state with + ``response.text()`` and ``response.messages`` populated. + ``model`` overrides the stored model id (useful for continuing + on a different model). + """ + return _response_from_dict(data, cls, model=model, async_=False) + + def on_done(self, callback): + "Register a callback to be called when the response is complete." + if not self._done: + self.done_callbacks.append(callback) + else: + callback(self) + + def _on_done(self): + for callback in self.done_callbacks: + callback(self) + + def __str__(self) -> str: + return self.text() + + def _force(self): + if not self._done: + list(self) + + def text(self) -> str: + "Return the full text of the response, executing the prompt if needed." + self._force() + return "".join(self._chunks) + + def text_or_raise(self) -> str: + return self.text() + + def execute_tool_calls( + self, + *, + before_call: Optional[BeforeCallSync] = None, + after_call: Optional[AfterCallSync] = None, + ) -> List[ToolResult]: + tool_results = [] + tools_by_name = {tool.name: tool for tool in self.prompt.tools} + + # Run prepare() on all Toolbox instances that need it + instances_to_prepare: list[Toolbox] = [] + for tool_to_prep in tools_by_name.values(): + inst = _get_instance(tool_to_prep.implementation) + if isinstance(inst, Toolbox) and not getattr(inst, "_prepared", False): + instances_to_prepare.append(inst) + + for inst in instances_to_prepare: + inst.prepare() + inst._prepared = True + + for tool_call in self.tool_calls(): + tool: Optional[Tool] = tools_by_name.get(tool_call.name) + # Tool could be None if the tool was not found in the prompt tools, + # but we still call the before_call method: + if before_call: + try: + cb_result = before_call(tool, tool_call) + if inspect.isawaitable(cb_result): + raise TypeError( + "Asynchronous 'before_call' callback provided to a synchronous tool execution context. " + "Please use an async chain/response or a synchronous callback." + ) + except CancelToolCall as ex: + tool_results.append( + ToolResult( + name=tool_call.name, + output="Cancelled: " + str(ex), + tool_call_id=tool_call.tool_call_id, + exception=ex, + ) + ) + continue + + if tool is None: + msg = 'tool "{}" does not exist'.format(tool_call.name) + tool_results.append( + ToolResult( + name=tool_call.name, + output="Error: " + msg, + tool_call_id=tool_call.tool_call_id, + exception=KeyError(msg), + ) + ) + continue + + if not tool.implementation: + raise ValueError( + "No implementation available for tool: {}".format(tool_call.name) + ) + + attachments = [] + exception = None + + try: + if inspect.iscoroutinefunction(tool.implementation): + result = asyncio.run(tool.implementation(**tool_call.arguments)) + else: + result = tool.implementation(**tool_call.arguments) + + if isinstance(result, ToolOutput): + attachments = result.attachments + result = result.output + + if not isinstance(result, str): + result = json.dumps(result, default=repr) + except Exception as ex: + result = f"Error: {ex}" + exception = ex + + tool_result_obj = ToolResult( + name=tool_call.name, + output=result, + attachments=attachments, + tool_call_id=tool_call.tool_call_id, + instance=_get_instance(tool.implementation), + exception=exception, + ) + + if after_call: + cb_result = after_call(tool, tool_call, tool_result_obj) + if inspect.isawaitable(cb_result): + raise TypeError( + "Asynchronous 'after_call' callback provided to a synchronous tool execution context. " + "Please use an async chain/response or a synchronous callback." + ) + tool_results.append(tool_result_obj) + return tool_results + + def tool_calls(self) -> List[ToolCall]: + "Return the list of tool calls made during this response." + self._force() + return self._tool_calls + + def tool_calls_or_raise(self) -> List[ToolCall]: + return self.tool_calls() + + def json(self) -> Optional[Dict[str, Any]]: + "Return the raw JSON response from the model, if available." + self._force() + return self.response_json + + def duration_ms(self) -> int: + self._force() + return int(((self._end or 0) - (self._start or 0)) * 1000) + + def datetime_utc(self) -> str: + self._force() + return self._start_utcnow.isoformat() if self._start_utcnow else "" + + def usage(self) -> Usage: + "Return token usage information for this response." + self._force() + return Usage( + input=self.input_tokens, + output=self.output_tokens, + details=self.token_details, + ) + + def _iter_events(self): + """Drive self.model.execute() once. Yields every chunk it + produces, each already appended to self._stream_events by + _process_chunk as a side effect. + """ + if isinstance(self.model, Model): + generator = self.model.execute( + self.prompt, + stream=self.stream, + response=self, + conversation=self.conversation, + ) + elif isinstance(self.model, KeyModel): + generator = self.model.execute( + self.prompt, + stream=self.stream, + response=self, + conversation=self.conversation, + key=self.model.get_key(self._key), + ) + else: + raise Exception("self.model must be a Model or KeyModel") + + for chunk in generator: + assert chunk is not None + yield chunk + + def __iter__(self) -> Iterator[str]: + self._start = time.monotonic() + self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) + if self._done: + yield from self._chunks + return + + for chunk in self._iter_events(): + text = self._process_chunk(chunk) + if text is not None: + yield text + + if self.conversation: + self.conversation.responses.append(self) + self._end = time.monotonic() + self._done = True + self._on_done() + + def stream_events(self): + """Yield StreamEvent objects as the model produces them. + + Whichever of __iter__ and stream_events runs first during live + streaming consumes the underlying generator. After completion, + both work — each replays from its own buffer. + """ + if self._done: + yield from self._stream_events + return + + self._start = time.monotonic() + self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) + for chunk in self._iter_events(): + # _process_chunk appends to self._stream_events; use it as + # the canonical source for what to yield so the replay path + # matches the live path byte-for-byte. + self._process_chunk(chunk) + yield self._stream_events[-1] + + if self.conversation: + self.conversation.responses.append(self) + self._end = time.monotonic() + self._done = True + self._on_done() + + @property + def messages(self) -> List[Any]: + """List of Message objects produced by this response. + + Almost always a single assistant Message; multiple messages are + possible for providers that emit multi-message responses during + server-side tool execution (not in this phase's scope). + + Responses rehydrated via ``Response.from_dict`` short-circuit + and return the stored messages directly. + """ + from .parts import Message + + loaded = getattr(self, "_loaded_messages", None) + if loaded is not None: + return list(loaded) + self._force() + parts = self._build_parts() + if not parts: + return [] + return [Message(role="assistant", parts=parts)] + + def __repr__(self): + text = "... not yet done ..." + if self._done: + text = "".join(self._chunks) + return "".format(self.prompt.prompt, text) + + +class AsyncResponse(_BaseResponse): + "Async response from a model." + + model: "AsyncModel" + conversation: Optional["AsyncConversation"] = None + + def reply( + self, + prompt: Optional[str] = None, + *, + messages: Optional[List[Any]] = None, + **kwargs, + ) -> "AsyncResponse": + """Async counterpart of Response.reply(). Requires this response + to have been awaited (so self.messages is available). + """ + from .parts import Message, TextPart + + if not self._done: + raise ValueError( + "Response not yet awaited — call `await response` before reply()" + ) + chain: List[Any] = list(self.prompt.messages) + list(self.messages) + if prompt: + chain.append( + Message(role="user", parts=[TextPart(text=prompt)]) + ) + if messages: + chain.extend(messages) + return self.model.prompt(messages=chain, **kwargs) + + def to_dict(self) -> ResponseDict: + """Async counterpart of Response.to_dict(). Requires awaiting.""" + if not self._done: + raise ValueError( + "Response not yet awaited — call `await response` before to_dict()" + ) + return _response_to_dict(self) + + @classmethod + def from_dict( + cls, + data: ResponseDict, + *, + model: Optional["AsyncModel"] = None, + ) -> "AsyncResponse": + """Async counterpart of Response.from_dict().""" + return _response_from_dict(data, cls, model=model, async_=True) + + @classmethod + def from_row(cls, db, row, _async=False): + return super().from_row(db, row, _async=True) + + async def on_done(self, callback): + "Register a callback to be called when the response is complete." + if not self._done: + self.done_callbacks.append(callback) + else: + if callable(callback): + # Ensure we handle both sync and async callbacks correctly + processed_callback = callback(self) + if inspect.isawaitable(processed_callback): + await processed_callback + elif inspect.isawaitable(callback): + await callback + + async def _on_done(self): + for callback_func in self.done_callbacks: + if callable(callback_func): + processed_callback = callback_func(self) + if inspect.isawaitable(processed_callback): + await processed_callback + elif inspect.isawaitable(callback_func): + await callback_func + + async def execute_tool_calls( + self, + *, + before_call: Optional[BeforeCallAsync] = None, + after_call: Optional[AfterCallAsync] = None, + ) -> List[ToolResult]: + tool_calls_list = await self.tool_calls() + tools_by_name = {tool.name: tool for tool in self.prompt.tools} + + # Run async prepare_async() on all Toolbox instances that need it + instances_to_prepare: list[Toolbox] = [] + for tool_to_prep in tools_by_name.values(): + inst = _get_instance(tool_to_prep.implementation) + if isinstance(inst, Toolbox) and not getattr( + inst, "_async_prepared", False + ): + instances_to_prepare.append(inst) + + for inst in instances_to_prepare: + await inst.prepare_async() + inst._async_prepared = True + + indexed_results: List[tuple[int, ToolResult]] = [] + async_tasks: List[asyncio.Task] = [] + + for idx, tc in enumerate(tool_calls_list): + tool: Optional[Tool] = tools_by_name.get(tc.name) + exception: Optional[Exception] = None + + if tool is None: + output = f'Error: tool "{tc.name}" does not exist' + exception = KeyError(tc.name) + elif not tool.implementation: + output = f'Error: tool "{tc.name}" has no implementation' + exception = KeyError(tc.name) + elif inspect.iscoroutinefunction(tool.implementation): + + async def run_async(tc=tc, tool=tool, idx=idx): + # before_call inside the task + if before_call: + try: + cb = before_call(tool, tc) + if inspect.isawaitable(cb): + await cb + except CancelToolCall as ex: + return idx, ToolResult( + name=tc.name, + output="Cancelled: " + str(ex), + tool_call_id=tc.tool_call_id, + exception=ex, + ) + + exception = None + attachments = [] + + try: + result = await tool.implementation(**tc.arguments) + if isinstance(result, ToolOutput): + attachments.extend(result.attachments) + result = result.output + output = ( + result + if isinstance(result, str) + else json.dumps(result, default=repr) + ) + except Exception as ex: + output = f"Error: {ex}" + exception = ex + + tr = ToolResult( + name=tc.name, + output=output, + attachments=attachments, + tool_call_id=tc.tool_call_id, + instance=_get_instance(tool.implementation), + exception=exception, + ) + + # after_call inside the task + if tool is not None and after_call: + cb2 = after_call(tool, tc, tr) + if inspect.isawaitable(cb2): + await cb2 + + return idx, tr + + async_tasks.append(asyncio.create_task(run_async())) + + else: + # Sync implementation: do hooks and call inline + if before_call: + try: + cb = before_call(tool, tc) + if inspect.isawaitable(cb): + await cb + except CancelToolCall as ex: + indexed_results.append( + ( + idx, + ToolResult( + name=tc.name, + output="Cancelled: " + str(ex), + tool_call_id=tc.tool_call_id, + exception=ex, + ), + ) + ) + continue + + exception = None + attachments = [] + + if tool is None: + output = f'Error: tool "{tc.name}" does not exist' + exception = KeyError(tc.name) + else: + try: + res = tool.implementation(**tc.arguments) + if inspect.isawaitable(res): + res = await res + if isinstance(res, ToolOutput): + attachments.extend(res.attachments) + res = res.output + output = ( + res + if isinstance(res, str) + else json.dumps(res, default=repr) + ) + except Exception as ex: + output = f"Error: {ex}" + exception = ex + + tr = ToolResult( + name=tc.name, + output=output, + attachments=attachments, + tool_call_id=tc.tool_call_id, + instance=_get_instance(tool.implementation), + exception=exception, + ) + + if tool is not None and after_call: + cb2 = after_call(tool, tc, tr) + if inspect.isawaitable(cb2): + await cb2 + + indexed_results.append((idx, tr)) + + # Await all async tasks in parallel + if async_tasks: + indexed_results.extend(await asyncio.gather(*async_tasks)) + + # Reorder by original index + indexed_results.sort(key=lambda x: x[0]) + return [tr for _, tr in indexed_results] + + def __aiter__(self): + self._start = time.monotonic() + self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) + if self._done: + self._iter_chunks = list(self._chunks) # Make a copy for iteration + return self + + def _ensure_async_generator(self): + if not hasattr(self, "_generator"): + if isinstance(self.model, AsyncModel): + self._generator = self.model.execute( + self.prompt, + stream=self.stream, + response=self, + conversation=self.conversation, + ) + elif isinstance(self.model, AsyncKeyModel): + self._generator = self.model.execute( + self.prompt, + stream=self.stream, + response=self, + conversation=self.conversation, + key=self.model.get_key(self._key), + ) + else: + raise ValueError("self.model must be an AsyncModel or AsyncKeyModel") + + async def _async_finalize(self): + if self.conversation: + self.conversation.responses.append(self) + self._end = time.monotonic() + self._done = True + if hasattr(self, "_generator"): + del self._generator + await self._on_done() + + async def __anext__(self) -> str: + if self._done: + if hasattr(self, "_iter_chunks") and self._iter_chunks: + return self._iter_chunks.pop(0) + raise StopAsyncIteration + + self._ensure_async_generator() + # Skip non-text events — iteration yields only text. Loop until + # we find a text chunk or the generator is exhausted. + while True: + try: + chunk = await self._generator.__anext__() + except StopAsyncIteration: + await self._async_finalize() + raise + assert chunk is not None + text = self._process_chunk(chunk) + if text is not None: + return text + + async def astream_events(self): + """Yield StreamEvent objects as the model produces them (async).""" + if self._done: + for event in self._stream_events: + yield event + return + + self._start = time.monotonic() + self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) + self._ensure_async_generator() + try: + while True: + try: + chunk = await self._generator.__anext__() + except StopAsyncIteration: + await self._async_finalize() + return + assert chunk is not None + self._process_chunk(chunk) + yield self._stream_events[-1] + finally: + pass + + @property + def messages(self) -> List[Any]: + """List of Message objects produced by this response. + + Raises ValueError if the response has not yet been awaited — + assembly depends on the full event stream. Responses rehydrated + via ``AsyncResponse.from_dict`` short-circuit and return the + stored messages. + """ + from .parts import Message + + loaded = getattr(self, "_loaded_messages", None) + if loaded is not None: + return list(loaded) + if not self._done: + raise ValueError( + "Response not yet awaited — use 'await response' first" + ) + parts = self._build_parts() + if not parts: + return [] + return [Message(role="assistant", parts=parts)] + + async def _force(self): + if not self._done: + temp_chunks = [] + async for chunk in self: + temp_chunks.append(chunk) + # This should populate self._chunks + return self + + def text_or_raise(self) -> str: + if not self._done: + raise ValueError("Response not yet awaited") + return "".join(self._chunks) + + async def text(self) -> str: + "Return the full text of the response, executing the prompt if needed." + await self._force() + return "".join(self._chunks) + + async def tool_calls(self) -> List[ToolCall]: + "Return the list of tool calls made during this response." + await self._force() + return self._tool_calls + + def tool_calls_or_raise(self) -> List[ToolCall]: + if not self._done: + raise ValueError("Response not yet awaited") + return self._tool_calls + + async def json(self) -> Optional[Dict[str, Any]]: + "Return the raw JSON response from the model, if available." + await self._force() + return self.response_json + + async def duration_ms(self) -> int: + await self._force() + return int(((self._end or 0) - (self._start or 0)) * 1000) + + async def datetime_utc(self) -> str: + await self._force() + return self._start_utcnow.isoformat() if self._start_utcnow else "" + + async def usage(self) -> Usage: + "Return token usage information for this response." + await self._force() + return Usage( + input=self.input_tokens, + output=self.output_tokens, + details=self.token_details, + ) + + def __await__(self): + return self._force().__await__() + + async def to_sync_response(self) -> Response: + await self._force() + # This conversion might be tricky if the model is AsyncModel, + # as Response expects a sync Model. For simplicity, we'll assume + # the primary use case is data transfer after completion. + # The model type on the new Response might need careful handling + # if it's intended for further execution. + # For now, let's assume self.model can be cast or is compatible. + sync_model = self.model + if not isinstance(self.model, (Model, KeyModel)): + # This is a placeholder. A proper conversion or shared base might be needed + # if the sync_response needs to be fully functional with its model. + # For now, we pass the async model, which might limit what sync_response can do. + pass + + response = Response( + self.prompt, + sync_model, # This might need adjustment based on how Model/AsyncModel relate + self.stream, + # conversation type needs to be compatible too. + conversation=( + self.conversation.to_sync_conversation() if self.conversation else None + ), + ) + response.id = self.id + response._chunks = list(self._chunks) # Copy chunks + response._done = self._done + response._end = self._end + response._start = self._start + response._start_utcnow = self._start_utcnow + response.input_tokens = self.input_tokens + response.output_tokens = self.output_tokens + response.token_details = self.token_details + response._prompt_json = self._prompt_json + response.response_json = self.response_json + response._tool_calls = list(self._tool_calls) + response.attachments = list(self.attachments) + response.resolved_model = self.resolved_model + return response + + @classmethod + def fake( + cls, + model: "AsyncModel", + prompt: str, + *attachments: List[Attachment], + system: str, + response: str, + ): + "Utility method to help with writing tests" + response_obj = cls( + model=model, + prompt=Prompt( + prompt, + model=model, + attachments=attachments, + system=system, + ), + stream=False, + ) + response_obj._done = True + response_obj._chunks = [response] + return response_obj + + def __repr__(self): + text = "... not yet awaited ..." + if self._done: + text = "".join(self._chunks) + return "".format(self.prompt.prompt, text) + + +def _chain_for_tool_results( + prior_response, tool_results, attachments +) -> List[Any]: + """Build the message chain for a tool-result turn in a chain loop. + + Takes the prior response's full input chain + its structured + output, then appends a tool-role message carrying the new + ToolResult outputs. Attachments (e.g. images returned by tools) + are folded into a subsequent user-role message. + + This is what gives ``response.prompt.messages`` on the tool- + result turn the complete history for the next provider call — + including any reasoning signatures or thoughtSignatures from the + prior turn. + """ + from .parts import ( + AttachmentPart, + Message, + TextPart, + ToolResultPart, + ) + + chain: List[Any] = list(prior_response.prompt.messages) + list( + prior_response.messages + ) + if tool_results: + chain.append( + Message( + role="tool", + parts=[ + ToolResultPart( + name=tr.name, + output=tr.output, + tool_call_id=tr.tool_call_id, + ) + for tr in tool_results + ], + ) + ) + # Attachments that came back from tools ride on a trailing user + # message (mimics the legacy attachments=[] kwarg behavior). + if attachments: + chain.append( + Message( + role="user", + parts=[AttachmentPart(attachment=a) for a in attachments], + ) + ) + return chain + + +class _BaseChainResponse: + prompt: "Prompt" + stream: bool + conversation: Optional["_BaseConversation"] = None + _key: Optional[str] = None + + def __init__( + self, + prompt: Prompt, + model: "_BaseModel", + stream: bool, + conversation: _BaseConversation, + key: Optional[str] = None, + chain_limit: Optional[int] = 10, + before_call: Optional[Union[BeforeCallSync, BeforeCallAsync]] = None, + after_call: Optional[Union[AfterCallSync, AfterCallAsync]] = None, + ): + self.prompt = prompt + self.model = model + self.stream = stream + self._key = key + self._responses: List[Any] = [] + self.conversation = conversation + self.chain_limit = chain_limit + self.before_call = before_call + self.after_call = after_call + + def log_to_db(self, db): + for response in self._responses: + if isinstance(response, AsyncResponse): + sync_response = asyncio.run(response.to_sync_response()) + elif isinstance(response, Response): + sync_response = response + else: + assert False, "Should have been a Response or AsyncResponse" + sync_response.log_to_db(db) + + +class ChainResponse(_BaseChainResponse): + _responses: List["Response"] + before_call: Optional[BeforeCallSync] = None + after_call: Optional[AfterCallSync] = None + + def responses(self) -> Iterator[Response]: + prompt = self.prompt + count = 0 + current_response: Optional[Response] = Response( + prompt, + self.model, + self.stream, + key=self._key, + conversation=self.conversation, + ) + while current_response: + count += 1 + yield current_response + self._responses.append(current_response) + if self.chain_limit and count >= self.chain_limit: + raise ValueError(f"Chain limit of {self.chain_limit} exceeded.") + + # This could raise llm.CancelToolCall: + tool_results = current_response.execute_tool_calls( + before_call=self.before_call, after_call=self.after_call + ) + attachments = [] + for tool_result in tool_results: + attachments.extend(tool_result.attachments) + if tool_results: + # Pre-bake the full chain for the tool-result turn so + # response.prompt.messages is what gets sent — carries + # thoughtSignatures, thinking signatures, and everything + # else the model needs for the next call. + next_chain = _chain_for_tool_results( + current_response, tool_results, attachments + ) + current_response = Response( + Prompt( + "", # Next prompt text is empty; tool_results drive it + self.model, + tools=current_response.prompt.tools, + tool_results=tool_results, + messages=next_chain, + # Carry system + system_fragments forward so + # stateless-per-turn adapters (OpenAI and + # friends that read prompt.system directly) + # keep seeing the system prompt on every call + # of the chain loop. + system=self.prompt._system, + system_fragments=self.prompt.system_fragments, + options=self.prompt.options, + attachments=attachments, + ), + self.model, + stream=self.stream, + key=self._key, + conversation=self.conversation, + ) + else: + current_response = None + break + + def __iter__(self) -> Iterator[str]: + for response_item in self.responses(): + yield from response_item + + def stream_events(self): + "Yield StreamEvents from every response in the chain." + for response_item in self.responses(): + yield from response_item.stream_events() + + def text(self) -> str: + return "".join(self) + + +class AsyncChainResponse(_BaseChainResponse): + _responses: List["AsyncResponse"] + before_call: Optional[BeforeCallAsync] = None + after_call: Optional[AfterCallAsync] = None + + async def responses(self) -> AsyncIterator[AsyncResponse]: + prompt = self.prompt + count = 0 + current_response: Optional[AsyncResponse] = AsyncResponse( + prompt, + self.model, + self.stream, + key=self._key, + conversation=self.conversation, + ) + while current_response: + count += 1 + yield current_response + self._responses.append(current_response) + + if self.chain_limit and count >= self.chain_limit: + raise ValueError(f"Chain limit of {self.chain_limit} exceeded.") + + # This could raise llm.CancelToolCall: + tool_results = await current_response.execute_tool_calls( + before_call=self.before_call, after_call=self.after_call + ) + if tool_results: + attachments = [] + for tool_result in tool_results: + attachments.extend(tool_result.attachments) + # Pre-bake chain so prompt.messages carries full history + # + any thinking/tool-call signatures from prior turn. + next_chain = _chain_for_tool_results( + current_response, tool_results, attachments + ) + prompt = Prompt( + "", + self.model, + tools=current_response.prompt.tools, + tool_results=tool_results, + messages=next_chain, + # Carry system + system_fragments forward — same + # reasoning as the sync path. + system=self.prompt._system, + system_fragments=self.prompt.system_fragments, + options=self.prompt.options, + attachments=attachments, + ) + current_response = AsyncResponse( + prompt, + self.model, + stream=self.stream, + key=self._key, + conversation=self.conversation, + ) + else: + current_response = None + break + + async def __aiter__(self) -> AsyncIterator[str]: + async for response_item in self.responses(): + async for chunk in response_item: + yield chunk + + async def astream_events(self): + "Yield StreamEvents from every response in the chain." + async for response_item in self.responses(): + async for event in response_item.astream_events(): + yield event + + async def text(self) -> str: + all_chunks = [] + async for chunk in self: + all_chunks.append(chunk) + return "".join(all_chunks) + + +class Options(BaseModel): + model_config = ConfigDict(extra="forbid") + + +_Options = Options + + +class _get_key_mixin: + needs_key: Optional[str] = None + key: Optional[str] = None + key_env_var: Optional[str] = None + + def get_key(self, explicit_key: Optional[str] = None) -> Optional[str]: + from llm import get_key + + if self.needs_key is None: + # This model doesn't use an API key + return None + + if self.key is not None: + # Someone already set model.key='...' + return self.key + + # Attempt to load a key using llm.get_key() + key_value = get_key( + explicit_key=explicit_key, + key_alias=self.needs_key, + env_var=self.key_env_var, + ) + if key_value: + return key_value + + # Show a useful error message + message = "No key found - add one using 'llm keys set {}'".format( + self.needs_key + ) + if self.key_env_var: + message += " or set the {} environment variable".format(self.key_env_var) + raise NeedsKeyException(message) + + +class _BaseModel(ABC, _get_key_mixin): + model_id: str + can_stream: bool = False + attachment_types: Set = set() + + supports_schema = False + supports_tools = False + + class Options(_Options): + pass + + def _validate_attachments( + self, attachments: Optional[List[Attachment]] = None + ) -> None: + if attachments and not self.attachment_types: + raise ValueError("This model does not support attachments") + for attachment in attachments or []: + attachment_type = attachment.resolve_type() + if attachment_type not in self.attachment_types: + raise ValueError( + f"This model does not support attachments of type '{attachment_type}', " + f"only {', '.join(self.attachment_types)}" + ) + + def __str__(self) -> str: + return "{}{}: {}".format( + self.__class__.__name__, + " (async)" if isinstance(self, (AsyncModel, AsyncKeyModel)) else "", + self.model_id, + ) + + def __repr__(self) -> str: + return f"<{str(self)}>" + + +class _Model(_BaseModel): + def conversation( + self, + tools: Optional[List[ToolDef]] = None, + before_call: Optional[BeforeCallSync] = None, + after_call: Optional[AfterCallSync] = None, + chain_limit: Optional[int] = None, + ) -> Conversation: + return Conversation( + model=self, + tools=tools, + before_call=before_call, + after_call=after_call, + chain_limit=chain_limit, + ) + + def prompt( + self, + prompt: Optional[str] = None, + *, + fragments: Optional[List[Union[str, Fragment]]] = None, + attachments: Optional[List[Attachment]] = None, + system: Optional[str] = None, + system_fragments: Optional[List[Union[str, Fragment]]] = None, + messages: Optional[List[Any]] = None, + stream: bool = True, + schema: Optional[Union[dict, type[BaseModel]]] = None, + tools: Optional[List[ToolDef]] = None, + tool_results: Optional[List[ToolResult]] = None, + **options, + ) -> Response: + key_value = options.pop("key", None) + self._validate_attachments(attachments) + return Response( + Prompt( + prompt, + fragments=fragments, + attachments=attachments, + system=system, + schema=schema, + tools=tools, + tool_results=tool_results, + system_fragments=system_fragments, + messages=messages, + model=self, + options=self.Options(**options), + ), + self, + stream, + key=key_value, + ) + + def chain( + self, + prompt: Optional[str] = None, + *, + fragments: Optional[List[str]] = None, + attachments: Optional[List[Attachment]] = None, + system: Optional[str] = None, + system_fragments: Optional[List[str]] = None, + messages: Optional[List[Any]] = None, + stream: bool = True, + schema: Optional[Union[dict, type[BaseModel]]] = None, + tools: Optional[List[ToolDef]] = None, + tool_results: Optional[List[ToolResult]] = None, + before_call: Optional[BeforeCallSync] = None, + after_call: Optional[AfterCallSync] = None, + key: Optional[str] = None, + options: Optional[dict] = None, + ) -> ChainResponse: + return self.conversation().chain( + prompt=prompt, + fragments=fragments, + attachments=attachments, + system=system, + system_fragments=system_fragments, + messages=messages, + stream=stream, + schema=schema, + tools=tools, + tool_results=tool_results, + before_call=before_call, + after_call=after_call, + key=key, + options=options, + ) + + +class Model(_Model): + @abstractmethod + def execute( + self, + prompt: Prompt, + stream: bool, + response: Response, + conversation: Optional[Conversation], + ) -> Iterator[str]: + pass + + +class KeyModel(_Model): + @abstractmethod + def execute( + self, + prompt: Prompt, + stream: bool, + response: Response, + conversation: Optional[Conversation], + key: Optional[str], + ) -> Iterator[str]: + pass + + +class _AsyncModel(_BaseModel): + def conversation( + self, + tools: Optional[List[ToolDef]] = None, + before_call: Optional[BeforeCallAsync] = None, + after_call: Optional[AfterCallAsync] = None, + chain_limit: Optional[int] = None, + ) -> AsyncConversation: + return AsyncConversation( + model=self, + tools=tools, + before_call=before_call, + after_call=after_call, + chain_limit=chain_limit, + ) + + def prompt( + self, + prompt: Optional[str] = None, + *, + fragments: Optional[List[Union[str, Fragment]]] = None, + attachments: Optional[List[Attachment]] = None, + system: Optional[str] = None, + schema: Optional[Union[dict, type[BaseModel]]] = None, + tools: Optional[List[ToolDef]] = None, + tool_results: Optional[List[ToolResult]] = None, + system_fragments: Optional[List[Union[str, Fragment]]] = None, + messages: Optional[List[Any]] = None, + stream: bool = True, + **options, + ) -> AsyncResponse: + key_value = options.pop("key", None) + self._validate_attachments(attachments) + return AsyncResponse( + Prompt( + prompt, + fragments=fragments, + attachments=attachments, + system=system, + schema=schema, + tools=tools, + tool_results=tool_results, + system_fragments=system_fragments, + messages=messages, + model=self, + options=self.Options(**options), + ), + self, + stream, + key=key_value, + ) + + def chain( + self, + prompt: Optional[str] = None, + *, + fragments: Optional[List[str]] = None, + attachments: Optional[List[Attachment]] = None, + system: Optional[str] = None, + system_fragments: Optional[List[str]] = None, + messages: Optional[List[Any]] = None, + stream: bool = True, + schema: Optional[Union[dict, type[BaseModel]]] = None, + tools: Optional[List[ToolDef]] = None, + tool_results: Optional[List[ToolResult]] = None, + before_call: Optional[BeforeCallAsync] = None, + after_call: Optional[AfterCallAsync] = None, + key: Optional[str] = None, + options: Optional[dict] = None, + ) -> AsyncChainResponse: + return self.conversation().chain( + prompt=prompt, + fragments=fragments, + attachments=attachments, + system=system, + system_fragments=system_fragments, + messages=messages, + stream=stream, + schema=schema, + tools=tools, + tool_results=tool_results, + before_call=before_call, + after_call=after_call, + key=key, + options=options, + ) + + +class AsyncModel(_AsyncModel): + @abstractmethod + async def execute( + self, + prompt: Prompt, + stream: bool, + response: AsyncResponse, + conversation: Optional[AsyncConversation], + ) -> AsyncGenerator[str, None]: + if False: # Ensure it's a generator type + yield "" + pass + + +class AsyncKeyModel(_AsyncModel): + @abstractmethod + async def execute( + self, + prompt: Prompt, + stream: bool, + response: AsyncResponse, + conversation: Optional[AsyncConversation], + key: Optional[str], + ) -> AsyncGenerator[str, None]: + if False: # Ensure it's a generator type + yield "" + pass + + +class EmbeddingModel(ABC, _get_key_mixin): + model_id: str + key: Optional[str] = None + needs_key: Optional[str] = None + key_env_var: Optional[str] = None + supports_text: bool = True + supports_binary: bool = False + batch_size: Optional[int] = None + + def _check(self, item: Union[str, bytes]): + if not self.supports_binary and isinstance(item, bytes): + raise ValueError( + "This model does not support binary data, only text strings" + ) + if not self.supports_text and isinstance(item, str): + raise ValueError( + "This model does not support text strings, only binary data" + ) + + def embed(self, item: Union[str, bytes]) -> List[float]: + "Embed a single text string or binary blob, return a list of floats" + self._check(item) + return next(iter(self.embed_batch([item]))) + + def embed_multi( + self, items: Iterable[Union[str, bytes]], batch_size: Optional[int] = None + ) -> Iterator[List[float]]: + "Embed multiple items in batches according to the model batch_size" + iter_items = iter(items) + effective_batch_size = self.batch_size if batch_size is None else batch_size + if (not self.supports_binary) or (not self.supports_text): + + def checking_iter(inner_items): + for item_to_check in inner_items: + self._check(item_to_check) + yield item_to_check + + iter_items = checking_iter(items) + if effective_batch_size is None: + yield from self.embed_batch(iter_items) + return + while True: + batch_items = list(islice(iter_items, effective_batch_size)) + if not batch_items: + break + yield from self.embed_batch(batch_items) + + @abstractmethod + def embed_batch(self, items: Iterable[Union[str, bytes]]) -> Iterator[List[float]]: + """ + Embed a batch of strings or blobs, return a list of lists of floats + """ + pass + + def __str__(self) -> str: + return "{}: {}".format(self.__class__.__name__, self.model_id) + + def __repr__(self) -> str: + return f"<{str(self)}>" + + +@dataclass +class ModelWithAliases: + "A model with its optional async counterpart and aliases." + + model: Model + async_model: AsyncModel + aliases: Set[str] + + def matches(self, query: str) -> bool: + query_lower = query.lower() + all_strings: List[str] = [] + all_strings.extend(self.aliases) + if self.model: + all_strings.append(str(self.model)) + if self.async_model: + all_strings.append(str(self.async_model.model_id)) + return any(query_lower in alias.lower() for alias in all_strings) + + +@dataclass +class EmbeddingModelWithAliases: + model: EmbeddingModel + aliases: Set[str] + + def matches(self, query: str) -> bool: + query_lower = query.lower() + all_strings: List[str] = [] + all_strings.extend(self.aliases) + all_strings.append(str(self.model)) + return any(query_lower in alias.lower() for alias in all_strings) + + +def _conversation_name(text): + # Collapse whitespace, including newlines + text = re.sub(r"\s+", " ", text) + if len(text) <= CONVERSATION_NAME_LENGTH: + return text + return text[: CONVERSATION_NAME_LENGTH - 1] + "…" + + +def _ensure_dict_schema(schema): + """Convert a Pydantic model to a JSON schema dict if needed.""" + if schema and not isinstance(schema, dict) and issubclass(schema, BaseModel): + schema_dict = schema.model_json_schema() + _remove_titles_recursively(schema_dict) + return schema_dict + return schema + + +def _remove_titles_recursively(obj): + """Recursively remove all 'title' fields from a nested dictionary.""" + if isinstance(obj, dict): + # Remove title if present + obj.pop("title", None) + + # Recursively process all values + for value in obj.values(): + _remove_titles_recursively(value) + elif isinstance(obj, list): + # Process each item in lists + for item in obj: + _remove_titles_recursively(item) + + +def _get_instance(implementation): + if hasattr(implementation, "__self__"): + return implementation.__self__ + return None diff --git a/build/lib/llm/parts.py b/build/lib/llm/parts.py new file mode 100644 index 000000000..e627f876b --- /dev/null +++ b/build/lib/llm/parts.py @@ -0,0 +1,340 @@ +"""Part, Message, and StreamEvent value types. + +Parts represent the structured content of model interactions: text, +reasoning, tool calls, tool results, and attachments. A Message wraps a +list of Parts with a role. StreamEvent wraps a streaming chunk with type +information so consumers can distinguish text from reasoning from tool +call fragments as they arrive. + +These types are pure values — identity (ids, parent links, storage keys) +is a storage concern that lives elsewhere. Two Messages with identical +content are equal. +""" + +import base64 +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from .models import Attachment +from .serialization import ( + AttachmentDict, + AttachmentPartDict, + MessageDict, + PartDict, + ReasoningPartDict, + TextPartDict, + ToolCallPartDict, + ToolResultPartDict, +) + + +def _attachment_to_dict(att: Attachment) -> AttachmentDict: + d: Dict[str, Any] = {} + if att.type: + d["type"] = att.type + if att.url: + d["url"] = att.url + if att.path: + d["path"] = att.path + if att.content: + d["content"] = base64.b64encode(att.content).decode("ascii") + return d # type: ignore[return-value] + + +def _attachment_from_dict(d: AttachmentDict) -> Attachment: + content = d.get("content") + if isinstance(content, str): + content = base64.b64decode(content) + return Attachment( + type=d.get("type"), + path=d.get("path"), + url=d.get("url"), + content=content, + ) + + +@dataclass +class Part: + """Base class for all parts. Role lives on the enclosing Message.""" + + def to_dict(self) -> PartDict: + raise NotImplementedError + + @staticmethod + def from_dict(d: PartDict) -> "Part": + type_ = d.get("type") + pm = d.get("provider_metadata") + if type_ == "text": + return TextPart(text=d.get("text", ""), provider_metadata=pm) + if type_ == "reasoning": + return ReasoningPart( + text=d.get("text", ""), + redacted=d.get("redacted", False), + token_count=d.get("token_count"), + provider_metadata=pm, + ) + if type_ == "tool_call": + return ToolCallPart( + name=d["name"], + arguments=d.get("arguments", {}), + tool_call_id=d.get("tool_call_id"), + server_executed=d.get("server_executed", False), + provider_metadata=pm, + ) + if type_ == "tool_result": + return ToolResultPart( + name=d["name"], + output=d.get("output", ""), + tool_call_id=d.get("tool_call_id"), + server_executed=d.get("server_executed", False), + exception=d.get("exception"), + attachments=[ + _attachment_from_dict(a) for a in d.get("attachments", []) + ], + provider_metadata=pm, + ) + if type_ == "attachment": + att_dict = d.get("attachment") + attachment = _attachment_from_dict(att_dict) if att_dict else None + return AttachmentPart(attachment=attachment, provider_metadata=pm) + raise ValueError(f"Unknown part type: {type_!r}") + + +@dataclass +class TextPart(Part): + text: str = "" + provider_metadata: Optional[Dict[str, Any]] = None + + def to_dict(self) -> TextPartDict: + d: Dict[str, Any] = {"type": "text", "text": self.text} + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d # type: ignore[return-value] + + +@dataclass +class ReasoningPart(Part): + """Reasoning/thinking tokens from the model. + + `redacted=True, text=""` represents the opaque-token-count case + (OpenAI GPT-5 series, Gemini) where the provider reports only a + count, not content. + """ + + text: str = "" + redacted: bool = False + token_count: Optional[int] = None + provider_metadata: Optional[Dict[str, Any]] = None + + def to_dict(self) -> ReasoningPartDict: + d: Dict[str, Any] = {"type": "reasoning", "text": self.text} + if self.redacted: + d["redacted"] = True + if self.token_count is not None: + d["token_count"] = self.token_count + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d # type: ignore[return-value] + + +@dataclass +class ToolCallPart(Part): + """A request by the model to call a tool. + + `server_executed=True` marks calls the provider executed on the + server (Anthropic web search, Gemini code execution) rather than + the LLM tool framework. + """ + + name: str = "" + arguments: Dict[str, Any] = field(default_factory=dict) + tool_call_id: Optional[str] = None + server_executed: bool = False + provider_metadata: Optional[Dict[str, Any]] = None + + def to_dict(self) -> ToolCallPartDict: + d: Dict[str, Any] = { + "type": "tool_call", + "name": self.name, + "arguments": self.arguments, + } + if self.tool_call_id is not None: + d["tool_call_id"] = self.tool_call_id + if self.server_executed: + d["server_executed"] = True + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d # type: ignore[return-value] + + +@dataclass +class ToolResultPart(Part): + """The result of a tool call.""" + + name: str = "" + output: str = "" + tool_call_id: Optional[str] = None + server_executed: bool = False + attachments: List[Any] = field(default_factory=list) + exception: Optional[str] = None + provider_metadata: Optional[Dict[str, Any]] = None + + def to_dict(self) -> ToolResultPartDict: + d: Dict[str, Any] = { + "type": "tool_result", + "name": self.name, + "output": self.output, + } + if self.tool_call_id is not None: + d["tool_call_id"] = self.tool_call_id + if self.server_executed: + d["server_executed"] = True + if self.exception is not None: + d["exception"] = self.exception + if self.attachments: + d["attachments"] = [_attachment_to_dict(a) for a in self.attachments] + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d # type: ignore[return-value] + + +@dataclass +class AttachmentPart(Part): + """An inline attachment (image, audio, file).""" + + attachment: Optional[Attachment] = None + provider_metadata: Optional[Dict[str, Any]] = None + + def to_dict(self) -> AttachmentPartDict: + d: Dict[str, Any] = {"type": "attachment"} + if self.attachment: + d["attachment"] = _attachment_to_dict(self.attachment) + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d # type: ignore[return-value] + + +@dataclass +class Message: + """A single turn in a conversation: role + list of parts. + + `parts` contains one or more Part objects. `provider_metadata` + carries opaque provider-specific data attached to the message as a + whole; part-level data lives on the individual Part's + `provider_metadata`. + """ + + role: str + parts: List[Part] = field(default_factory=list) + provider_metadata: Optional[Dict[str, Any]] = None + + def to_dict(self) -> MessageDict: + d: Dict[str, Any] = { + "role": self.role, + "parts": [p.to_dict() for p in self.parts], + } + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d # type: ignore[return-value] + + @staticmethod + def from_dict(d: MessageDict) -> "Message": + return Message( + role=d["role"], + parts=[Part.from_dict(p) for p in d.get("parts", [])], + provider_metadata=d.get("provider_metadata"), + ) + + +def normalize_parts(items: Any) -> List[Part]: + """Normalize helper inputs to a list of Part objects. + + Accepts str (→ TextPart), Attachment (→ AttachmentPart), Part + (passed through), or a list/tuple of those (flattened one level). + """ + out: List[Part] = [] + for item in items: + if isinstance(item, Part): + out.append(item) + elif isinstance(item, str): + out.append(TextPart(text=item)) + elif isinstance(item, Attachment): + out.append(AttachmentPart(attachment=item)) + elif isinstance(item, (list, tuple)): + out.extend(normalize_parts(item)) + else: + raise TypeError(f"Cannot convert {item!r} to an llm Part") + return out + + +def system( + *items: Any, provider_metadata: Optional[Dict[str, Any]] = None +) -> Message: + "Build a Message with role='system'." + return Message( + role="system", + parts=normalize_parts(items), + provider_metadata=provider_metadata, + ) + + +def user( + *items: Any, provider_metadata: Optional[Dict[str, Any]] = None +) -> Message: + "Build a Message with role='user'." + return Message( + role="user", + parts=normalize_parts(items), + provider_metadata=provider_metadata, + ) + + +def assistant( + *items: Any, provider_metadata: Optional[Dict[str, Any]] = None +) -> Message: + "Build a Message with role='assistant'." + return Message( + role="assistant", + parts=normalize_parts(items), + provider_metadata=provider_metadata, + ) + + +def tool_message( + *items: Any, provider_metadata: Optional[Dict[str, Any]] = None +) -> Message: + "Build a Message with role='tool' (typically wrapping ToolResultParts)." + return Message( + role="tool", + parts=normalize_parts(items), + provider_metadata=provider_metadata, + ) + + +@dataclass +class StreamEvent: + """A streaming event from a model response. + + `part_index` groups events into parts — events sharing an index + belong to the same logical part. Mixing families (e.g. text with + tool_call_name) at the same index is a plugin bug. + + `provider_metadata` carries opaque provider data (Anthropic + `signature`, Gemini `thoughtSignature`, OpenAI `encrypted_content`) + that must be echoed back on the next request; the framework merges + it onto the finalized Part (last non-None wins per top-level key). + + `message_index` is for providers that emit multiple assistant + messages in a single response (Anthropic server-side tool + execution); most plugins leave it at 0. + """ + + type: str # "text" / "reasoning" / "tool_call_name" / + # "tool_call_args" / "tool_result" + chunk: str + part_index: int + tool_call_id: Optional[str] = None + server_executed: bool = False + tool_name: Optional[str] = None + provider_metadata: Optional[Dict[str, Any]] = None + message_index: int = 0 diff --git a/build/lib/llm/plugins.py b/build/lib/llm/plugins.py new file mode 100644 index 000000000..0125ede04 --- /dev/null +++ b/build/lib/llm/plugins.py @@ -0,0 +1,50 @@ +import importlib +from importlib import metadata +import os +import pluggy +import sys +from . import hookspecs + +DEFAULT_PLUGINS = ( + "llm.default_plugins.openai_models", + "llm.default_plugins.default_tools", +) + +pm = pluggy.PluginManager("llm") +pm.add_hookspecs(hookspecs) + +LLM_LOAD_PLUGINS = os.environ.get("LLM_LOAD_PLUGINS", None) + +_loaded = False + + +def load_plugins(): + global _loaded + if _loaded: + return + _loaded = True + if not hasattr(sys, "_called_from_test") and LLM_LOAD_PLUGINS is None: + # Only load plugins if not running tests + pm.load_setuptools_entrypoints("llm") + + # Load any plugins specified in LLM_LOAD_PLUGINS") + if LLM_LOAD_PLUGINS is not None: + for package_name in [ + name for name in LLM_LOAD_PLUGINS.split(",") if name.strip() + ]: + try: + distribution = metadata.distribution(package_name) # Updated call + llm_entry_points = [ + ep for ep in distribution.entry_points if ep.group == "llm" + ] + for entry_point in llm_entry_points: + mod = entry_point.load() + pm.register(mod, name=entry_point.name) + # Ensure name can be found in plugin_to_distinfo later: + pm._plugin_distinfo.append((mod, distribution)) # type: ignore + except metadata.PackageNotFoundError: + sys.stderr.write(f"Plugin {package_name} could not be found\n") + + for plugin in DEFAULT_PLUGINS: + mod = importlib.import_module(plugin) + pm.register(mod, plugin) diff --git a/build/lib/llm/py.typed b/build/lib/llm/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/build/lib/llm/serialization.py b/build/lib/llm/serialization.py new file mode 100644 index 000000000..b6b4bf972 --- /dev/null +++ b/build/lib/llm/serialization.py @@ -0,0 +1,182 @@ +"""TypedDict spec for the JSON-safe wire form of Part, Message, and Response. + +These are the exact shapes returned by ``Part.to_dict()``, +``Message.to_dict()``, and ``Response.to_dict()`` — and accepted by the +matching ``from_dict`` classmethods. They are the canonical wire format; +use them to annotate any code that reads or writes serialized llm data. + +Example:: + + from llm.serialization import MessageDict + + def save_messages(conn, messages: list[MessageDict]) -> None: + for m in messages: + conn.execute( + "INSERT INTO messages(role, parts_json) VALUES (?, ?)", + (m["role"], json.dumps(m["parts"])), + ) + +Or pair with Pydantic's TypeAdapter for runtime validation:: + + from pydantic import TypeAdapter + from llm.serialization import MessageDict + + msg = TypeAdapter(MessageDict).validate_python(incoming_dict) + +Or export JSON Schema for cross-language consumers:: + + schema = TypeAdapter(MessageDict).json_schema() + +The TypedDicts are erased at runtime — zero overhead. ``NotRequired`` +keys may be absent from a serialized payload; required keys must +always be present. +""" + +from typing import Any, Dict, List, Literal, Union + +# NotRequired moved to typing in 3.11; use typing_extensions for 3.10 +# support. typing_extensions is a transitive dep via pydantic. +from typing_extensions import NotRequired, TypedDict + + +__all__ = [ + "AttachmentDict", + "AttachmentPartDict", + "MessageDict", + "PartDict", + "PromptDict", + "ReasoningPartDict", + "ResponseDict", + "TextPartDict", + "ToolCallPartDict", + "ToolResultPartDict", + "UsageDict", +] + + +# ---- Attachment payload (nested inside AttachmentPartDict + tool results) ---- + + +class AttachmentDict(TypedDict, total=False): + """Nested attachment payload. All fields optional — an Attachment + may carry a type, a url, a path, and/or base64-encoded content. + """ + + type: str + url: str + path: str + # base64-encoded bytes when the attachment was constructed with raw + # content= bytes. + content: str + + +# ---- Per-Part TypedDicts (discriminated by the `type` field) ----------------- + + +class TextPartDict(TypedDict): + type: Literal["text"] + text: str + provider_metadata: NotRequired[Dict[str, Any]] + + +class ReasoningPartDict(TypedDict): + type: Literal["reasoning"] + text: str + # Redacted reasoning: text is "" and token_count carries the opaque + # count reported by the provider (OpenAI GPT-5, Gemini thinking). + redacted: NotRequired[bool] + token_count: NotRequired[int] + provider_metadata: NotRequired[Dict[str, Any]] + + +class ToolCallPartDict(TypedDict): + type: Literal["tool_call"] + name: str + arguments: Dict[str, Any] + tool_call_id: NotRequired[str] + # True for provider-executed calls (Anthropic web search, Gemini code + # execution). Client echoes the block back as-is on next turn. + server_executed: NotRequired[bool] + provider_metadata: NotRequired[Dict[str, Any]] + + +class ToolResultPartDict(TypedDict): + type: Literal["tool_result"] + name: str + output: str + tool_call_id: NotRequired[str] + server_executed: NotRequired[bool] + exception: NotRequired[str] + attachments: NotRequired[List[AttachmentDict]] + provider_metadata: NotRequired[Dict[str, Any]] + + +class AttachmentPartDict(TypedDict): + type: Literal["attachment"] + attachment: NotRequired[AttachmentDict] + provider_metadata: NotRequired[Dict[str, Any]] + + +PartDict = Union[ + TextPartDict, + ReasoningPartDict, + ToolCallPartDict, + ToolResultPartDict, + AttachmentPartDict, +] +"""Discriminated union of Part dict shapes. Use with +``pydantic.TypeAdapter(PartDict)`` to validate / dispatch by ``type``. +""" + + +# ---- Message ---------------------------------------------------------------- + + +class MessageDict(TypedDict): + """JSON-safe form of ``llm.Message``. + + ``role`` is one of "user", "assistant", "system", "tool" in practice + — typed as ``str`` here to leave room for provider-specific values. + """ + + role: str + parts: List[PartDict] + provider_metadata: NotRequired[Dict[str, Any]] + + +# ---- Response + nested shapes ----------------------------------------------- + + +class PromptDict(TypedDict): + """The ``prompt`` sub-dict of ``Response.to_dict()`` — captures the + full input chain that was sent for this turn plus any options that + apply.""" + + messages: List[MessageDict] + options: NotRequired[Dict[str, Any]] + system: NotRequired[str] + + +class UsageDict(TypedDict, total=False): + """Optional usage block on ``ResponseDict``. All fields optional; + providers vary in which they report.""" + + input: int + output: int + details: Dict[str, Any] + + +class ResponseDict(TypedDict): + """JSON-safe form of ``llm.Response`` — everything needed for + ``Response.from_dict`` to rehydrate and ``response.reply()`` to + continue a conversation across a process boundary. + """ + + model: str + prompt: PromptDict + messages: List[MessageDict] + # Audit fields — present on a freshly-serialized response, optional + # on hand-constructed ones. + id: NotRequired[str] + usage: NotRequired[UsageDict] + datetime_utc: NotRequired[str] diff --git a/build/lib/llm/templates.py b/build/lib/llm/templates.py new file mode 100644 index 000000000..ac1b7c716 --- /dev/null +++ b/build/lib/llm/templates.py @@ -0,0 +1,92 @@ +from pydantic import BaseModel, ConfigDict +import string +from typing import Optional, Any, Dict, List, Tuple + + +class AttachmentType(BaseModel): + type: str + value: str + + +class Template(BaseModel): + """A reusable prompt template.""" + + name: str + prompt: Optional[str] = None + system: Optional[str] = None + attachments: Optional[List[str]] = None + attachment_types: Optional[List[AttachmentType]] = None + model: Optional[str] = None + defaults: Optional[Dict[str, Any]] = None + options: Optional[Dict[str, Any]] = None + extract: Optional[bool] = None # For extracting fenced code blocks + extract_last: Optional[bool] = None + schema_object: Optional[dict] = None + fragments: Optional[List[str]] = None + system_fragments: Optional[List[str]] = None + tools: Optional[List[str]] = None + functions: Optional[str] = None + + model_config = ConfigDict(extra="forbid") + + class MissingVariables(Exception): + pass + + def __init__(self, **data): + super().__init__(**data) + # Not a pydantic field to avoid YAML being able to set it + # this controls if Python inline functions code is trusted + self._functions_is_trusted = False + + def evaluate( + self, input: str, params: Optional[Dict[str, Any]] = None + ) -> Tuple[Optional[str], Optional[str]]: + """Evaluate the template with the given input and parameters, returning (prompt, system).""" + params = params or {} + params["input"] = input + if self.defaults: + for k, v in self.defaults.items(): + if k not in params: + params[k] = v + prompt: Optional[str] = None + system: Optional[str] = None + if not self.prompt: + system = self.interpolate(self.system, params) + prompt = input + else: + prompt = self.interpolate(self.prompt, params) + system = self.interpolate(self.system, params) + return prompt, system + + def vars(self) -> set: + """Return the set of variable names used in the prompt and system templates.""" + all_vars = set() + for text in [self.prompt, self.system]: + if not text: + continue + all_vars.update(self.extract_vars(string.Template(text))) + return all_vars + + @classmethod + def interpolate(cls, text: Optional[str], params: Dict[str, Any]) -> Optional[str]: + """Substitute template variables in text with values from params, raising MissingVariables if any are absent.""" + if not text: + return text + # Confirm all variables in text are provided + string_template = string.Template(text) + vars = cls.extract_vars(string_template) + missing = [p for p in vars if p not in params] + if missing: + raise cls.MissingVariables( + "Missing variables: {}".format(", ".join(missing)) + ) + return string_template.substitute(**params) + + @staticmethod + def extract_vars(string_template: string.Template) -> List[str]: + """Extract and return the list of named variable identifiers from a string.Template.""" + return [ + match.group("named") + for match in string_template.pattern.finditer(string_template.template) + if match.group("named") + ] diff --git a/build/lib/llm/tools.py b/build/lib/llm/tools.py new file mode 100644 index 000000000..5ac0a7dcb --- /dev/null +++ b/build/lib/llm/tools.py @@ -0,0 +1,37 @@ +from datetime import datetime, timezone +from importlib.metadata import version +import time + + +def llm_version() -> str: + "Return the installed version of llm" + return version("llm") + + +def llm_time() -> dict: + "Returns the current time, as local time and UTC" + # Get current times + utc_time = datetime.now(timezone.utc) + local_time = datetime.now() + + # Get timezone information + local_tz_name = time.tzname[time.localtime().tm_isdst] + is_dst = bool(time.localtime().tm_isdst) + + # Calculate offset + offset_seconds = -time.timezone if not is_dst else -time.altzone + offset_hours = offset_seconds // 3600 + offset_minutes = (offset_seconds % 3600) // 60 + + timezone_offset = ( + f"UTC{'+' if offset_hours >= 0 else ''}{offset_hours:02d}:{offset_minutes:02d}" + ) + + return { + "utc_time": utc_time.strftime("%Y-%m-%d %H:%M:%S UTC"), + "utc_time_iso": utc_time.isoformat(), + "local_timezone": local_tz_name, + "local_time": local_time.strftime("%Y-%m-%d %H:%M:%S"), + "timezone_offset": timezone_offset, + "is_dst": is_dst, + } diff --git a/build/lib/llm/utils.py b/build/lib/llm/utils.py new file mode 100644 index 000000000..587f19284 --- /dev/null +++ b/build/lib/llm/utils.py @@ -0,0 +1,735 @@ +import click +import hashlib +import httpx +import itertools +import json +import pathlib +import puremagic +import re +import sqlite_utils +import textwrap +from typing import Any, List, Dict, Optional, Tuple, Type +import os +import threading +import time +from typing import Final + +from ulid import ULID + +MIME_TYPE_FIXES = { + "audio/wave": "audio/wav", +} + + +class Fragment(str): + def __new__(cls, content, *args, **kwargs): + # For immutable classes like str, __new__ creates the string object + return super().__new__(cls, content) + + def __init__(self, content, source=""): + # Initialize our custom attributes + self.source = source + + def id(self): + return hashlib.sha256(self.encode("utf-8")).hexdigest() + + +def mimetype_from_string(content) -> Optional[str]: + try: + type_ = puremagic.from_string(content, mime=True) + return MIME_TYPE_FIXES.get(type_, type_) + except puremagic.PureError: + return None + + +def mimetype_from_path(path) -> Optional[str]: + try: + type_ = puremagic.from_file(path, mime=True) + return MIME_TYPE_FIXES.get(type_, type_) + except puremagic.PureError: + return None + + +def dicts_to_table_string( + headings: List[str], dicts: List[Dict[str, str]] +) -> List[str]: + max_lengths = [len(h) for h in headings] + + # Compute maximum length for each column + for d in dicts: + for i, h in enumerate(headings): + if h in d and len(str(d[h])) > max_lengths[i]: + max_lengths[i] = len(str(d[h])) + + # Generate formatted table strings + res = [] + res.append(" ".join(h.ljust(max_lengths[i]) for i, h in enumerate(headings))) + + for d in dicts: + row = [] + for i, h in enumerate(headings): + row.append(str(d.get(h, "")).ljust(max_lengths[i])) + res.append(" ".join(row)) + + return res + + +def remove_dict_none_values(d): + """ + Recursively remove keys with value of None or value of a dict that is all values of None + """ + if not isinstance(d, dict): + return d + new_dict = {} + for key, value in d.items(): + if value is not None: + if isinstance(value, dict): + nested = remove_dict_none_values(value) + if nested: + new_dict[key] = nested + elif isinstance(value, list): + new_dict[key] = [remove_dict_none_values(v) for v in value] + else: + new_dict[key] = value + return new_dict + + +class _LogResponse(httpx.Response): + def iter_bytes(self, *args, **kwargs): + for chunk in super().iter_bytes(*args, **kwargs): + click.echo(chunk.decode(), err=True) + yield chunk + + +class _LogTransport(httpx.BaseTransport): + def __init__(self, transport: httpx.BaseTransport): + self.transport = transport + + def handle_request(self, request: httpx.Request) -> httpx.Response: + response = self.transport.handle_request(request) + return _LogResponse( + status_code=response.status_code, + headers=response.headers, + stream=response.stream, + extensions=response.extensions, + ) + + +def _no_accept_encoding(request: httpx.Request): + request.headers.pop("accept-encoding", None) + + +def _log_response(response: httpx.Response): + request = response.request + click.echo(f"Request: {request.method} {request.url}", err=True) + click.echo(" Headers:", err=True) + for key, value in request.headers.items(): + if key.lower() == "authorization": + value = "[...]" + if key.lower() == "cookie": + value = value.split("=")[0] + "=..." + click.echo(f" {key}: {value}", err=True) + click.echo(" Body:", err=True) + try: + request_body = json.loads(request.content) + click.echo( + textwrap.indent(json.dumps(request_body, indent=2), " "), err=True + ) + except json.JSONDecodeError: + click.echo(textwrap.indent(request.content.decode(), " "), err=True) + click.echo(f"Response: status_code={response.status_code}", err=True) + click.echo(" Headers:", err=True) + for key, value in response.headers.items(): + if key.lower() == "set-cookie": + value = value.split("=")[0] + "=..." + click.echo(f" {key}: {value}", err=True) + click.echo(" Body:", err=True) + + +def logging_client() -> httpx.Client: + return httpx.Client( + transport=_LogTransport(httpx.HTTPTransport()), + event_hooks={"request": [_no_accept_encoding], "response": [_log_response]}, + ) + + +def simplify_usage_dict(d): + # Recursively remove keys with value 0 and empty dictionaries + def remove_empty_and_zero(obj): + if isinstance(obj, dict): + cleaned = { + k: remove_empty_and_zero(v) + for k, v in obj.items() + if v != 0 and v != {} + } + return {k: v for k, v in cleaned.items() if v is not None and v != {}} + return obj + + return remove_empty_and_zero(d) or {} + + +def token_usage_string(input_tokens, output_tokens, token_details) -> str: + bits = [] + if input_tokens is not None: + bits.append(f"{format(input_tokens, ',')} input") + if output_tokens is not None: + bits.append(f"{format(output_tokens, ',')} output") + if token_details: + bits.append(json.dumps(token_details)) + return ", ".join(bits) + + +def extract_fenced_code_block(text: str, last: bool = False) -> Optional[str]: + """ + Extracts and returns Markdown fenced code block found in the given text. + + The function handles fenced code blocks that: + - Use at least three backticks (`). + - May include a language tag immediately after the opening backticks. + - Use more than three backticks as long as the closing fence has the same number. + + If no fenced code block is found, the function returns None. + + Args: + text (str): The input text to search for a fenced code block. + last (bool): Extract the last code block if True, otherwise the first. + + Returns: + Optional[str]: The content of the fenced code block, or None if not found. + """ + # Regex pattern to match fenced code blocks + # - ^ or \n ensures that the fence is at the start of a line + # - (`{3,}) captures the opening backticks (at least three) + # - (\w+)? optionally captures the language tag + # - \n matches the newline after the opening fence + # - (.*?) non-greedy match for the code block content + # - (?P=fence) ensures that the closing fence has the same number of backticks + # - [ ]* allows for optional spaces between the closing fence and newline + # - (?=\n|$) ensures that the closing fence is followed by a newline or end of string + pattern = re.compile( + r"""(?m)^(?P`{3,})(?P\w+)?\n(?P.*?)^(?P=fence)[ ]*(?=\n|$)""", + re.DOTALL, + ) + matches = list(pattern.finditer(text)) + if matches: + match = matches[-1] if last else matches[0] + return match.group("code") + return None + + +def make_schema_id(schema: dict) -> Tuple[str, str]: + schema_json = json.dumps(schema, separators=(",", ":")) + schema_id = hashlib.blake2b(schema_json.encode(), digest_size=16).hexdigest() + return schema_id, schema_json + + +def output_rows_as_json(rows, nl=False, compact=False, json_cols=()): + """ + Output rows as JSON - either newline-delimited or an array + + Parameters: + - rows: Iterable of dictionaries to output + - nl: Boolean, if True, use newline-delimited JSON + - compact: Boolean, if True uses [{"...": "..."}\n {"...": "..."}] format + - json_cols: Iterable of columns that contain JSON + + Yields: + - Stream of strings to be output + """ + current_iter, next_iter = itertools.tee(rows, 2) + next(next_iter, None) + first = True + + for row, next_row in itertools.zip_longest(current_iter, next_iter): + is_last = next_row is None + for col in json_cols: + row[col] = json.loads(row[col]) + + if nl: + # Newline-delimited JSON: one JSON object per line + yield json.dumps(row) + elif compact: + # Compact array format: [{"...": "..."}\n {"...": "..."}] + yield "{firstchar}{serialized}{maybecomma}{lastchar}".format( + firstchar="[" if first else " ", + serialized=json.dumps(row), + maybecomma="," if not is_last else "", + lastchar="]" if is_last else "", + ) + else: + # Pretty-printed array format with indentation + yield "{firstchar}{serialized}{maybecomma}{lastchar}".format( + firstchar="[\n" if first else "", + serialized=textwrap.indent(json.dumps(row, indent=2), " "), + maybecomma="," if not is_last else "", + lastchar="\n]" if is_last else "", + ) + first = False + + if first and not nl: + # We didn't output any rows, so yield the empty list + yield "[]" + + +def resolve_schema_input(db, schema_input, load_template): + # schema_input might be JSON or a filepath or an ID or t:name + if not schema_input: + return + if schema_input.strip().startswith("t:"): + name = schema_input.strip()[2:] + schema_object = None + try: + template = load_template(name) + schema_object = template.schema_object + except ValueError: + raise click.ClickException("Invalid template: {}".format(name)) + if not schema_object: + raise click.ClickException("Template '{}' has no schema".format(name)) + return template.schema_object + if schema_input.strip().startswith("{"): + try: + return json.loads(schema_input) + except ValueError: + pass + if " " in schema_input.strip() or "," in schema_input: + # Treat it as schema DSL + return schema_dsl(schema_input) + # Is it a file on disk? + path = pathlib.Path(schema_input) + if path.exists(): + try: + return json.loads(path.read_text()) + except ValueError: + raise click.ClickException("Schema file contained invalid JSON") + # Last attempt: is it an ID in the DB? + try: + row = db["schemas"].get(schema_input) + return json.loads(row["content"]) + except (sqlite_utils.db.NotFoundError, ValueError): + raise click.BadParameter("Invalid schema") + + +def schema_summary(schema: dict) -> str: + """ + Extract property names from a JSON schema and format them in a + concise way that highlights the array/object structure. + + Args: + schema (dict): A JSON schema dictionary + + Returns: + str: A human-friendly summary of the schema structure + """ + if not schema or not isinstance(schema, dict): + return "" + + schema_type = schema.get("type", "") + + if schema_type == "object": + props = schema.get("properties", {}) + prop_summaries = [] + + for name, prop_schema in props.items(): + prop_type = prop_schema.get("type", "") + + if prop_type == "array": + items = prop_schema.get("items", {}) + items_summary = schema_summary(items) + prop_summaries.append(f"{name}: [{items_summary}]") + elif prop_type == "object": + nested_summary = schema_summary(prop_schema) + prop_summaries.append(f"{name}: {nested_summary}") + else: + prop_summaries.append(name) + + return "{" + ", ".join(prop_summaries) + "}" + + elif schema_type == "array": + items = schema.get("items", {}) + return schema_summary(items) + + return "" + + +def schema_dsl(schema_dsl: str, multi: bool = False) -> Dict[str, Any]: + """ + Build a JSON schema from a concise schema string. + + Args: + schema_dsl: A string representing a schema in the concise format. + Can be comma-separated or newline-separated. + multi: Boolean, return a schema for an "items" array of these + + Returns: + A dictionary representing the JSON schema. + """ + # Type mapping dictionary + type_mapping = { + "int": "integer", + "float": "number", + "bool": "boolean", + "str": "string", + } + + # Initialize the schema dictionary with required elements + json_schema: Dict[str, Any] = {"type": "object", "properties": {}, "required": []} + + # Check if the schema is newline-separated or comma-separated + if "\n" in schema_dsl: + fields = [field.strip() for field in schema_dsl.split("\n") if field.strip()] + else: + fields = [field.strip() for field in schema_dsl.split(",") if field.strip()] + + # Process each field + for field in fields: + # Extract field name, type, and description + if ":" in field: + field_info, description = field.split(":", 1) + description = description.strip() + else: + field_info = field + description = "" + + # Process field name and type + field_parts = field_info.strip().split() + field_name = field_parts[0].strip() + + # Default type is string + field_type = "string" + + # If type is specified, use it + if len(field_parts) > 1: + type_indicator = field_parts[1].strip() + if type_indicator in type_mapping: + field_type = type_mapping[type_indicator] + + # Add field to properties + json_schema["properties"][field_name] = {"type": field_type} + + # Add description if provided + if description: + json_schema["properties"][field_name]["description"] = description + + # Add field to required list + json_schema["required"].append(field_name) + + if multi: + return multi_schema(json_schema) + else: + return json_schema + + +def multi_schema(schema: dict) -> dict: + "Wrap JSON schema in an 'items': [] array" + return { + "type": "object", + "properties": {"items": {"type": "array", "items": schema}}, + "required": ["items"], + } + + +def find_unused_key(item: dict, key: str) -> str: + 'Return unused key, e.g. for {"id": "1"} and key "id" returns "id_"' + while key in item: + key += "_" + return key + + +def truncate_string( + text: str, + max_length: int = 100, + normalize_whitespace: bool = False, + keep_end: bool = False, +) -> str: + """ + Truncate a string to a maximum length, with options to normalize whitespace and keep both start and end. + + Args: + text: The string to truncate + max_length: Maximum length of the result string + normalize_whitespace: If True, replace all whitespace with a single space + keep_end: If True, keep both beginning and end of string + + Returns: + Truncated string + """ + if not text: + return text + + if normalize_whitespace: + text = re.sub(r"\s+", " ", text) + + if len(text) <= max_length: + return text + + # Minimum sensible length for keep_end is 9 characters: "a... z" + min_keep_end_length = 9 + + if keep_end and max_length >= min_keep_end_length: + # Calculate how much text to keep at each end + # Subtract 5 for the "... " separator + cutoff = (max_length - 5) // 2 + return text[:cutoff] + "... " + text[-cutoff:] + else: + # Fall back to simple truncation for very small max_length + return text[: max_length - 3] + "..." + + +def ensure_fragment(db, content): + sql = """ + insert into fragments (hash, content, datetime_utc, source) + values (:hash, :content, datetime('now'), :source) + on conflict(hash) do nothing + """ + hash_id = hashlib.sha256(content.encode("utf-8")).hexdigest() + source = None + if isinstance(content, Fragment): + source = content.source + with db.conn: + db.execute(sql, {"hash": hash_id, "content": content, "source": source}) + return list( + db.query("select id from fragments where hash = :hash", {"hash": hash_id}) + )[0]["id"] + + +def ensure_tool(db, tool): + sql = """ + insert into tools (hash, name, description, input_schema, plugin) + values (:hash, :name, :description, :input_schema, :plugin) + on conflict(hash) do nothing + """ + with db.conn: + db.execute( + sql, + { + "hash": tool.hash(), + "name": tool.name, + "description": tool.description, + "input_schema": json.dumps(tool.input_schema), + "plugin": tool.plugin, + }, + ) + return list( + db.query("select id from tools where hash = :hash", {"hash": tool.hash()}) + )[0]["id"] + + +def maybe_fenced_code(content: str) -> str: + "Return the content as a fenced code block if it looks like code" + is_code = False + if content.count("<") > 10: + is_code = True + if not is_code: + # Are 90% of the lines under 120 chars? + lines = content.splitlines() + if len(lines) > 3: + num_short = sum(1 for line in lines if len(line) < 120) + if num_short / len(lines) > 0.9: + is_code = True + if is_code: + # Find number of backticks not already present + num_backticks = 3 + while "`" * num_backticks in content: + num_backticks += 1 + # Add backticks + content = ( + "\n" + + "`" * num_backticks + + "\n" + + content.strip() + + "\n" + + "`" * num_backticks + ) + return content + + +_plugin_prefix_re = re.compile(r"^[a-zA-Z0-9_-]+:") + + +def has_plugin_prefix(value: str) -> bool: + "Check if value starts with alphanumeric prefix followed by a colon" + return bool(_plugin_prefix_re.match(value)) + + +def _parse_kwargs(arg_str: str) -> Dict[str, Any]: + """Parse key=value pairs where each value is valid JSON.""" + tokens = [] + buf = [] + depth = 0 + in_string = False + string_char = "" + escape = False + + for ch in arg_str: + if in_string: + buf.append(ch) + if escape: + escape = False + elif ch == "\\": + escape = True + elif ch == string_char: + in_string = False + else: + if ch in "\"'": + in_string = True + string_char = ch + buf.append(ch) + elif ch in "{[(": + depth += 1 + buf.append(ch) + elif ch in "}])": + depth -= 1 + buf.append(ch) + elif ch == "," and depth == 0: + tokens.append("".join(buf).strip()) + buf = [] + else: + buf.append(ch) + if buf: + tokens.append("".join(buf).strip()) + + kwargs: Dict[str, Any] = {} + for token in tokens: + if not token: + continue + if "=" not in token: + raise ValueError(f"Invalid keyword spec segment: '{token}'") + key, value_str = token.split("=", 1) + key = key.strip() + value_str = value_str.strip() + try: + value = json.loads(value_str) + except json.JSONDecodeError as e: + raise ValueError(f"Value for '{key}' is not valid JSON: {value_str}") from e + kwargs[key] = value + return kwargs + + +def instantiate_from_spec(class_map: Dict[str, Type], spec: str): + """ + Instantiate a class from a specification string with flexible argument formats. + + This function parses a specification string that defines a class name and its + constructor arguments, then instantiates the class using the provided class + mapping. The specification supports multiple argument formats for flexibility. + + Parameters + ---------- + class_map : Dict[str, Type] + A mapping from class names (strings) to their corresponding class objects. + Only classes present in this mapping can be instantiated. + spec : str + A specification string defining the class to instantiate and its arguments. + + Format: "ClassName" or "ClassName(arguments)" + + Supported argument formats: + - Empty: ClassName() - calls constructor with no arguments + - JSON object: ClassName({"key": "value", "other": 42}) - unpacked as **kwargs + - Single JSON value: ClassName("hello") or ClassName([1,2,3]) - passed as single positional argument + - Key-value pairs: ClassName(name="test", count=5, items=[1,2]) - parsed as individual kwargs + where values must be valid JSON + + Returns + ------- + object + An instance of the specified class, constructed with the parsed arguments. + + Raises + ------ + ValueError + If the spec string format is invalid, if the class name is not found in + class_map, if JSON parsing fails, or if argument parsing encounters errors. + """ + m = re.fullmatch(r"\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?:\((.*)\))?\s*$", spec) + if not m: + raise ValueError(f"Invalid spec string: '{spec}'") + class_name, arg_body = m.group(1), (m.group(2) or "").strip() + if class_name not in class_map: + raise ValueError(f"Unknown class '{class_name}'") + + cls = class_map[class_name] + + # No arguments at all + if arg_body == "": + return cls() + + # Starts with { -> JSON object to kwargs + if arg_body.lstrip().startswith("{"): + try: + kw = json.loads(arg_body) + except json.JSONDecodeError as e: + raise ValueError("Argument JSON object is not valid JSON") from e + if not isinstance(kw, dict): + raise ValueError("Top-level JSON must be an object when using {} form") + return cls(**kw) + + # Starts with quote / number / [ / t f n for single positional JSON value + if re.match(r'\s*(["\[\d\-]|true|false|null)', arg_body, re.I): + try: + positional_value = json.loads(arg_body) + except json.JSONDecodeError as e: + raise ValueError("Positional argument must be valid JSON") from e + return cls(positional_value) + + # Otherwise treat as key=value pairs + kwargs = _parse_kwargs(arg_body) + return cls(**kwargs) + + +NANOSECS_IN_MILLISECS = 1000000 +TIMESTAMP_LEN = 6 +RANDOMNESS_LEN = 10 + +_lock: Final = threading.Lock() +_last: Optional[bytes] = None # 16-byte last produced ULID + + +def monotonic_ulid() -> ULID: + """ + Return a ULID instance that is guaranteed to be *strictly larger* than every + other ULID returned by this function inside the same process. + + It works the same way the reference JavaScript `monotonicFactory` does: + * If the current call happens in the same millisecond as the previous + one, the 80-bit randomness part is incremented by exactly one. + * As soon as the system clock moves forward, a brand-new ULID with + cryptographically secure randomness is generated. + * If more than 2**80 ULIDs are requested within a single millisecond + an `OverflowError` is raised (practically impossible). + """ + global _last + + now_ms = time.time_ns() // NANOSECS_IN_MILLISECS + + with _lock: + # First call + if _last is None: + _last = _fresh(now_ms) + return ULID(_last) + + # Decode timestamp from the last ULID we handed out + last_ms = int.from_bytes(_last[:TIMESTAMP_LEN], "big") + + # If the millisecond is the same, increment the randomness + if now_ms == last_ms: + rand_int = int.from_bytes(_last[TIMESTAMP_LEN:], "big") + 1 + if rand_int >= 1 << (RANDOMNESS_LEN * 8): + raise OverflowError( + "Randomness overflow: > 2**80 ULIDs requested " + "in one millisecond!" + ) + randomness = rand_int.to_bytes(RANDOMNESS_LEN, "big") + _last = _last[:TIMESTAMP_LEN] + randomness + return ULID(_last) + + # New millisecond, start fresh + _last = _fresh(now_ms) + return ULID(_last) + + +def _fresh(ms: int) -> bytes: + """Build a brand-new 16-byte ULID for the given millisecond.""" + timestamp = int.to_bytes(ms, TIMESTAMP_LEN, "big") + randomness = os.urandom(RANDOMNESS_LEN) + return timestamp + randomness diff --git a/docs/help.md b/docs/help.md index a5a63566e..fa774ba33 100644 --- a/docs/help.md +++ b/docs/help.md @@ -144,6 +144,7 @@ Options: --no-stream Do not stream output -n, --no-log Don't log to database --log Log prompt and response to the database + -R, --no-reasoning Don't display reasoning output -c, --continue Continue the most recent conversation. --cid, --conversation TEXT Continue the conversation with the given ID. --key TEXT API key to use @@ -175,6 +176,7 @@ Options: -o, --option ... key/value options for the model -d, --database FILE Path to log database --no-stream Do not stream output + -R, --no-reasoning Don't display reasoning output --key TEXT API key to use -T, --tool TEXT Name of a tool to make available to the model --functions TEXT Python code block or file path defining From 564889512594554e7bc573b50e1a1a774f899faf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Apr 2026 19:58:39 -0700 Subject: [PATCH 032/258] Remove accidentally committed build/ directory and gitignore it Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 1 + build/lib/llm/__init__.py | 515 --- build/lib/llm/__main__.py | 4 - build/lib/llm/cli.py | 4094 ----------------- build/lib/llm/default_plugins/__init__.py | 0 .../lib/llm/default_plugins/default_tools.py | 8 - .../lib/llm/default_plugins/openai_models.py | 1212 ----- build/lib/llm/embeddings.py | 367 -- build/lib/llm/embeddings_migrations.py | 89 - build/lib/llm/errors.py | 6 - build/lib/llm/hookspecs.py | 35 - build/lib/llm/migrations.py | 420 -- build/lib/llm/models.py | 2966 ------------ build/lib/llm/parts.py | 340 -- build/lib/llm/plugins.py | 50 - build/lib/llm/py.typed | 0 build/lib/llm/serialization.py | 182 - build/lib/llm/templates.py | 92 - build/lib/llm/tools.py | 37 - build/lib/llm/utils.py | 735 --- 20 files changed, 1 insertion(+), 11152 deletions(-) delete mode 100644 build/lib/llm/__init__.py delete mode 100644 build/lib/llm/__main__.py delete mode 100644 build/lib/llm/cli.py delete mode 100644 build/lib/llm/default_plugins/__init__.py delete mode 100644 build/lib/llm/default_plugins/default_tools.py delete mode 100644 build/lib/llm/default_plugins/openai_models.py delete mode 100644 build/lib/llm/embeddings.py delete mode 100644 build/lib/llm/embeddings_migrations.py delete mode 100644 build/lib/llm/errors.py delete mode 100644 build/lib/llm/hookspecs.py delete mode 100644 build/lib/llm/migrations.py delete mode 100644 build/lib/llm/models.py delete mode 100644 build/lib/llm/parts.py delete mode 100644 build/lib/llm/plugins.py delete mode 100644 build/lib/llm/py.typed delete mode 100644 build/lib/llm/serialization.py delete mode 100644 build/lib/llm/templates.py delete mode 100644 build/lib/llm/tools.py delete mode 100644 build/lib/llm/utils.py diff --git a/.gitignore b/.gitignore index aa1fee1f0..7c5ff42e6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ venv .eggs .pytest_cache *.egg-info +build/ .DS_Store .idea/ .vscode/ diff --git a/build/lib/llm/__init__.py b/build/lib/llm/__init__.py deleted file mode 100644 index bb84c3911..000000000 --- a/build/lib/llm/__init__.py +++ /dev/null @@ -1,515 +0,0 @@ -from .hookspecs import hookimpl -from .errors import ( - ModelError, - NeedsKeyException, -) -from .models import ( - AsyncConversation, - AsyncKeyModel, - AsyncModel, - AsyncResponse, - Attachment, - CancelToolCall, - Conversation, - EmbeddingModel, - EmbeddingModelWithAliases, - KeyModel, - Model, - ModelWithAliases, - Options, - Prompt, - Response, - Tool, - Toolbox, - ToolCall, - ToolOutput, - ToolResult, - Usage, -) -from .parts import ( - AttachmentPart, - Message, - Part, - ReasoningPart, - StreamEvent, - TextPart, - ToolCallPart, - ToolResultPart, - assistant, - system, - tool_message, - user, -) -from .utils import schema_dsl, Fragment -from .embeddings import Collection -from .templates import Template -from .plugins import pm, load_plugins -import click -from typing import Any, Dict, List, Optional, Callable, Type, Union -import inspect -import json -import os -import pathlib -import struct - -__all__ = [ - "AsyncConversation", - "AsyncKeyModel", - "AsyncModel", - "AsyncResponse", - "assistant", - "Attachment", - "AttachmentPart", - "CancelToolCall", - "Collection", - "Conversation", - "Fragment", - "get_async_model", - "get_key", - "get_model", - "hookimpl", - "KeyModel", - "Message", - "Model", - "ModelError", - "NeedsKeyException", - "Options", - "Part", - "Prompt", - "ReasoningPart", - "Response", - "schema_dsl", - "StreamEvent", - "system", - "Template", - "TextPart", - "Tool", - "Toolbox", - "ToolCall", - "ToolCallPart", - "tool_message", - "ToolOutput", - "ToolResult", - "ToolResultPart", - "Usage", - "user", - "user_dir", -] -DEFAULT_MODEL = "gpt-4o-mini" - - -def get_plugins(all=False): - plugins = [] - plugin_to_distinfo = dict(pm.list_plugin_distinfo()) - for plugin in pm.get_plugins(): - if not all and plugin.__name__.startswith("llm.default_plugins."): - continue - plugin_info = { - "name": plugin.__name__, - "hooks": [h.name for h in pm.get_hookcallers(plugin)], - } - distinfo = plugin_to_distinfo.get(plugin) - if distinfo: - plugin_info["version"] = distinfo.version - plugin_info["name"] = ( - getattr(distinfo, "name", None) or distinfo.project_name - ) - plugins.append(plugin_info) - return plugins - - -def get_models_with_aliases() -> List["ModelWithAliases"]: - model_aliases = [] - - # Include aliases from aliases.json - aliases_path = user_dir() / "aliases.json" - extra_model_aliases: Dict[str, list] = {} - if aliases_path.exists(): - configured_aliases = json.loads(aliases_path.read_text()) - for alias, model_id in configured_aliases.items(): - extra_model_aliases.setdefault(model_id, []).append(alias) - - def register(model, async_model=None, aliases=None): - alias_list = list(aliases or []) - if model.model_id in extra_model_aliases: - alias_list.extend(extra_model_aliases[model.model_id]) - model_aliases.append(ModelWithAliases(model, async_model, alias_list)) - - load_plugins() - pm.hook.register_models(register=register, model_aliases=model_aliases) - - return model_aliases - - -def _get_loaders(hook_method) -> Dict[str, Callable]: - load_plugins() - loaders = {} - - def register(prefix, loader): - suffix = 0 - prefix_to_try = prefix - while prefix_to_try in loaders: - suffix += 1 - prefix_to_try = f"{prefix}_{suffix}" - loaders[prefix_to_try] = loader - - hook_method(register=register) - return loaders - - -def get_template_loaders() -> Dict[str, Callable[[str], Template]]: - """Get template loaders registered by plugins.""" - return _get_loaders(pm.hook.register_template_loaders) - - -def get_fragment_loaders() -> Dict[ - str, - Callable[[str], Union[Fragment, Attachment, List[Union[Fragment, Attachment]]]], -]: - """Get fragment loaders registered by plugins.""" - return _get_loaders(pm.hook.register_fragment_loaders) - - -def get_tools() -> Dict[str, Union[Tool, Type[Toolbox]]]: - """Return all tools (llm.Tool and llm.Toolbox) registered by plugins.""" - load_plugins() - tools: Dict[str, Union[Tool, Type[Toolbox]]] = {} - - # Variable to track current plugin name - current_plugin_name = None - - def register( - tool_or_function: Union[Tool, Type[Toolbox], Callable[..., Any]], - name: Optional[str] = None, - ) -> None: - tool: Union[Tool, Type[Toolbox], None] = None - - # If it's a Toolbox class, set the plugin field on it - if inspect.isclass(tool_or_function): - if issubclass(tool_or_function, Toolbox): - tool = tool_or_function - if current_plugin_name: - tool.plugin = current_plugin_name - tool.name = name or tool.__name__ - else: - raise TypeError( - "Toolbox classes must inherit from llm.Toolbox, {} does not.".format( - tool_or_function.__name__ - ) - ) - - # If it's already a Tool instance, use it directly - elif isinstance(tool_or_function, Tool): - tool = tool_or_function - if name: - tool.name = name - if current_plugin_name: - tool.plugin = current_plugin_name - - # If it's a bare function, wrap it in a Tool - else: - tool = Tool.function(tool_or_function, name=name) - if current_plugin_name: - tool.plugin = current_plugin_name - - # Get the name for the tool/toolbox - if tool: - # For Toolbox classes, use their name attribute or class name - if inspect.isclass(tool) and issubclass(tool, Toolbox): - prefix = name or getattr(tool, "name", tool.__name__) or "" - else: - prefix = name or tool.name or "" - - suffix = 0 - candidate = prefix - - # Avoid name collisions - while candidate in tools: - suffix += 1 - candidate = f"{prefix}_{suffix}" - - tools[candidate] = tool - - # Call each plugin's register_tools hook individually to track current_plugin_name - for plugin in pm.get_plugins(): - current_plugin_name = pm.get_name(plugin) - hook_caller = pm.hook.register_tools - plugin_impls = [ - impl for impl in hook_caller.get_hookimpls() if impl.plugin is plugin - ] - for impl in plugin_impls: - impl.function(register=register) - - return tools - - -def get_embedding_models_with_aliases() -> List["EmbeddingModelWithAliases"]: - model_aliases = [] - - # Include aliases from aliases.json - aliases_path = user_dir() / "aliases.json" - extra_model_aliases: Dict[str, list] = {} - if aliases_path.exists(): - configured_aliases = json.loads(aliases_path.read_text()) - for alias, model_id in configured_aliases.items(): - extra_model_aliases.setdefault(model_id, []).append(alias) - - def register(model, aliases=None): - alias_list = list(aliases or []) - if model.model_id in extra_model_aliases: - alias_list.extend(extra_model_aliases[model.model_id]) - model_aliases.append(EmbeddingModelWithAliases(model, alias_list)) - - load_plugins() - pm.hook.register_embedding_models(register=register) - - return model_aliases - - -def get_embedding_models(): - models = [] - - def register(model, aliases=None): - models.append(model) - - load_plugins() - pm.hook.register_embedding_models(register=register) - return models - - -def get_embedding_model(name): - aliases = get_embedding_model_aliases() - try: - return aliases[name] - except KeyError: - raise UnknownModelError("Unknown model: " + str(name)) - - -def get_embedding_model_aliases() -> Dict[str, EmbeddingModel]: - model_aliases = {} - for model_with_aliases in get_embedding_models_with_aliases(): - for alias in model_with_aliases.aliases: - model_aliases[alias] = model_with_aliases.model - model_aliases[model_with_aliases.model.model_id] = model_with_aliases.model - return model_aliases - - -def get_async_model_aliases() -> Dict[str, AsyncModel]: - async_model_aliases = {} - for model_with_aliases in get_models_with_aliases(): - if model_with_aliases.async_model: - for alias in model_with_aliases.aliases: - async_model_aliases[alias] = model_with_aliases.async_model - async_model_aliases[model_with_aliases.model.model_id] = ( - model_with_aliases.async_model - ) - return async_model_aliases - - -def get_model_aliases() -> Dict[str, Model]: - model_aliases = {} - for model_with_aliases in get_models_with_aliases(): - if model_with_aliases.model: - for alias in model_with_aliases.aliases: - model_aliases[alias] = model_with_aliases.model - model_aliases[model_with_aliases.model.model_id] = model_with_aliases.model - return model_aliases - - -class UnknownModelError(KeyError): - pass - - -def get_models() -> List[Model]: - "Get all registered models" - models_with_aliases = get_models_with_aliases() - return [mwa.model for mwa in models_with_aliases if mwa.model] - - -def get_async_models() -> List[AsyncModel]: - "Get all registered async models" - models_with_aliases = get_models_with_aliases() - return [mwa.async_model for mwa in models_with_aliases if mwa.async_model] - - -def get_async_model(name: Optional[str] = None) -> AsyncModel: - "Get an async model by name or alias" - aliases = get_async_model_aliases() - name = name or get_default_model() - try: - return aliases[name] - except KeyError: - # Does a sync model exist? - sync_model = None - try: - sync_model = get_model(name, _skip_async=True) - except UnknownModelError: - pass - if sync_model: - raise UnknownModelError("Unknown async model (sync model exists): " + name) - else: - raise UnknownModelError("Unknown model: " + name) - - -def get_model(name: Optional[str] = None, _skip_async: bool = False) -> Model: - "Get a model by name or alias" - aliases = get_model_aliases() - name = name or get_default_model() - try: - return aliases[name] - except KeyError: - # Does an async model exist? - if _skip_async: - raise UnknownModelError("Unknown model: " + name) - async_model = None - try: - async_model = get_async_model(name) - except UnknownModelError: - pass - if async_model: - raise UnknownModelError("Unknown model (async model exists): " + name) - else: - raise UnknownModelError("Unknown model: " + name) - - -def get_key( - explicit_key: Optional[str] = None, - key_alias: Optional[str] = None, - env_var: Optional[str] = None, - *, - alias: Optional[str] = None, - env: Optional[str] = None, - input: Optional[str] = None, -) -> Optional[str]: - """ - Return an API key based on a hierarchy of potential sources. You should use the keyword arguments, - the positional arguments are here purely for backwards-compatibility with older code. - - :param input: Input provided by the user. This may be the key, or an alias of a key in keys.json. - :param alias: The alias used to retrieve the key from the keys.json file. - :param env: Name of the environment variable to check for the key as a final fallback. - """ - if alias: - key_alias = alias - if env: - env_var = env - if input: - explicit_key = input - stored_keys = load_keys() - # If user specified an alias, use the key stored for that alias - if explicit_key in stored_keys: - return stored_keys[explicit_key] - if explicit_key: - # User specified a key that's not an alias, use that - return explicit_key - # Stored key over-rides environment variables over-ride the default key - if key_alias in stored_keys: - return stored_keys[key_alias] - # Finally try environment variable - if env_var and os.environ.get(env_var): - return os.environ[env_var] - # Couldn't find it - return None - - -def load_keys(): - path = user_dir() / "keys.json" - if path.exists(): - return json.loads(path.read_text()) - else: - return {} - - -def user_dir(): - llm_user_path = os.environ.get("LLM_USER_PATH") - if llm_user_path: - path = pathlib.Path(llm_user_path) - else: - path = pathlib.Path(click.get_app_dir("io.datasette.llm")) - path.mkdir(exist_ok=True, parents=True) - return path - - -def set_alias(alias, model_id_or_alias): - """ - Set an alias to point to the specified model. - """ - path = user_dir() / "aliases.json" - path.parent.mkdir(parents=True, exist_ok=True) - if not path.exists(): - path.write_text("{}\n") - try: - current = json.loads(path.read_text()) - except json.decoder.JSONDecodeError: - # We're going to write a valid JSON file in a moment: - current = {} - # Resolve model_id_or_alias to a model_id - try: - model = get_model(model_id_or_alias) - model_id = model.model_id - except UnknownModelError: - # Try to resolve it to an embedding model - try: - model = get_embedding_model(model_id_or_alias) - model_id = model.model_id - except UnknownModelError: - # Set the alias to the exact string they provided instead - model_id = model_id_or_alias - current[alias] = model_id - path.write_text(json.dumps(current, indent=4) + "\n") - - -def remove_alias(alias): - """ - Remove an alias. - """ - path = user_dir() / "aliases.json" - if not path.exists(): - raise KeyError("No aliases.json file exists") - try: - current = json.loads(path.read_text()) - except json.decoder.JSONDecodeError: - raise KeyError("aliases.json file is not valid JSON") - if alias not in current: - raise KeyError("No such alias: {}".format(alias)) - del current[alias] - path.write_text(json.dumps(current, indent=4) + "\n") - - -def encode(values): - return struct.pack("<" + "f" * len(values), *values) - - -def decode(binary): - return struct.unpack("<" + "f" * (len(binary) // 4), binary) - - -def cosine_similarity(a, b): - dot_product = sum(x * y for x, y in zip(a, b)) - magnitude_a = sum(x * x for x in a) ** 0.5 - magnitude_b = sum(x * x for x in b) ** 0.5 - return dot_product / (magnitude_a * magnitude_b) - - -def get_default_model(filename="default_model.txt", default=DEFAULT_MODEL): - path = user_dir() / filename - if path.exists(): - return path.read_text().strip() - else: - return default - - -def set_default_model(model, filename="default_model.txt"): - path = user_dir() / filename - if model is None and path.exists(): - path.unlink() - else: - path.write_text(model) - - -def get_default_embedding_model(): - return get_default_model("default_embedding_model.txt", None) - - -def set_default_embedding_model(model): - set_default_model(model, "default_embedding_model.txt") diff --git a/build/lib/llm/__main__.py b/build/lib/llm/__main__.py deleted file mode 100644 index 98dcca0c2..000000000 --- a/build/lib/llm/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .cli import cli - -if __name__ == "__main__": - cli() diff --git a/build/lib/llm/cli.py b/build/lib/llm/cli.py deleted file mode 100644 index f5cdd586d..000000000 --- a/build/lib/llm/cli.py +++ /dev/null @@ -1,4094 +0,0 @@ -import asyncio -import click -from click_default_group import DefaultGroup -from dataclasses import asdict -from importlib.metadata import version -import io -import json -import os -from llm import ( - Attachment, - AsyncConversation, - AsyncKeyModel, - AsyncResponse, - CancelToolCall, - Collection, - Conversation, - Fragment, - Response, - Template, - Tool, - Toolbox, - UnknownModelError, - KeyModel, - encode, - get_async_model, - get_default_model, - get_default_embedding_model, - get_embedding_models_with_aliases, - get_embedding_model_aliases, - get_embedding_model, - get_plugins, - get_tools, - get_fragment_loaders, - get_template_loaders, - get_model, - get_model_aliases, - get_models_with_aliases, - user_dir, - set_alias, - set_default_model, - set_default_embedding_model, - remove_alias, -) -from llm.models import _BaseConversation, ChainResponse - -from .migrations import migrate -from .plugins import pm, load_plugins -from .utils import ( - ensure_fragment, - extract_fenced_code_block, - find_unused_key, - has_plugin_prefix, - instantiate_from_spec, - make_schema_id, - maybe_fenced_code, - mimetype_from_path, - mimetype_from_string, - multi_schema, - output_rows_as_json, - resolve_schema_input, - schema_dsl, - schema_summary, - token_usage_string, - truncate_string, -) -import base64 -import httpx -import inspect -import pathlib -import pydantic -import re -import readline -from runpy import run_module -import shutil -import sqlite_utils -from sqlite_utils.utils import rows_from_file, Format -import sys -import textwrap -from typing import cast, Dict, Optional, Iterable, List, Union, Tuple, Type, Any -import warnings -import yaml - -warnings.simplefilter("ignore", ResourceWarning) - -DEFAULT_TEMPLATE = "prompt: " - - -class FragmentNotFound(Exception): - pass - - -def display_stream_events(events, *, show_reasoning=True): - """Consume a sync iterator of StreamEvents and write them. - - Text events go to stdout. Reasoning events go to stderr in dim style. - A newline is written to stderr at each reasoning→text transition so - the assistant text starts on a fresh visual line. - """ - was_reasoning = False - for event in events: - if event.type == "text": - if was_reasoning and show_reasoning: - click.echo("", err=True) - was_reasoning = False - click.echo(event.chunk, nl=False) - elif event.type == "reasoning" and show_reasoning: - was_reasoning = True - click.echo(click.style(event.chunk, dim=True), nl=False, err=True) - - -async def display_async_stream_events(events, *, show_reasoning=True): - """Async counterpart of display_stream_events.""" - was_reasoning = False - async for event in events: - if event.type == "text": - if was_reasoning and show_reasoning: - click.echo("", err=True) - was_reasoning = False - click.echo(event.chunk, nl=False) - elif event.type == "reasoning" and show_reasoning: - was_reasoning = True - click.echo(click.style(event.chunk, dim=True), nl=False, err=True) - - -def validate_fragment_alias(ctx, param, value): - if not re.match(r"^[a-zA-Z0-9_-]+$", value): - raise click.BadParameter("Fragment alias must be alphanumeric") - return value - - -def resolve_fragments( - db: sqlite_utils.Database, fragments: Iterable[str], allow_attachments: bool = False -) -> List[Union[Fragment, Attachment]]: - """ - Resolve fragment strings into a mixed of llm.Fragment() and llm.Attachment() objects. - """ - - def _load_by_alias(fragment: str) -> Tuple[Optional[str], Optional[str]]: - rows = list( - db.query( - """ - select content, source from fragments - left join fragment_aliases on fragments.id = fragment_aliases.fragment_id - where alias = :alias or hash = :alias limit 1 - """, - {"alias": fragment}, - ) - ) - if rows: - row = rows[0] - return row["content"], row["source"] - return None, None - - # The fragment strings could be URLs or paths or plugin references - resolved: List[Union[Fragment, Attachment]] = [] - for fragment in fragments: - if fragment.startswith("http://") or fragment.startswith("https://"): - llm_version = version("llm") - headers = {"User-Agent": f"llm/{llm_version} (https://llm.datasette.io/)"} - client = httpx.Client( - follow_redirects=True, max_redirects=3, headers=headers - ) - response = client.get(fragment) - response.raise_for_status() - resolved.append(Fragment(response.text, fragment)) - elif fragment == "-": - resolved.append(Fragment(sys.stdin.read(), "-")) - elif has_plugin_prefix(fragment): - prefix, rest = fragment.split(":", 1) - loaders = get_fragment_loaders() - if prefix not in loaders: - raise FragmentNotFound("Unknown fragment prefix: {}".format(prefix)) - loader = loaders[prefix] - try: - result = loader(rest) - if not isinstance(result, list): - result = [result] - if not allow_attachments and any( - isinstance(r, Attachment) for r in result - ): - raise FragmentNotFound( - "Fragment loader {} returned a disallowed attachment".format( - prefix - ) - ) - resolved.extend(result) - except Exception as ex: - raise FragmentNotFound( - "Could not load fragment {}: {}".format(fragment, ex) - ) - else: - # Try from the DB - content, source = _load_by_alias(fragment) - if content is not None: - resolved.append(Fragment(content, source)) - else: - # Now try path - path = pathlib.Path(fragment) - if path.exists(): - resolved.append(Fragment(path.read_text(), str(path.resolve()))) - else: - raise FragmentNotFound(f"Fragment '{fragment}' not found") - return resolved - - -def process_fragments_in_chat( - db: sqlite_utils.Database, prompt: str -) -> tuple[str, list[Fragment], list[Attachment]]: - """ - Process any !fragment commands in a chat prompt and return the modified prompt plus resolved fragments and attachments. - """ - prompt_lines = [] - fragments = [] - attachments = [] - for line in prompt.splitlines(): - if line.startswith("!fragment "): - try: - fragment_strs = line.strip().removeprefix("!fragment ").split() - fragments_and_attachments = resolve_fragments( - db, fragments=fragment_strs, allow_attachments=True - ) - fragments += [ - fragment - for fragment in fragments_and_attachments - if isinstance(fragment, Fragment) - ] - attachments += [ - attachment - for attachment in fragments_and_attachments - if isinstance(attachment, Attachment) - ] - except FragmentNotFound as ex: - raise click.ClickException(str(ex)) - else: - prompt_lines.append(line) - return "\n".join(prompt_lines), fragments, attachments - - -class AttachmentError(Exception): - """Exception raised for errors in attachment resolution.""" - - pass - - -def resolve_attachment(value): - """ - Resolve an attachment from a string value which could be: - - "-" for stdin - - A URL - - A file path - - Returns an Attachment object. - Raises AttachmentError if the attachment cannot be resolved. - """ - if value == "-": - content = sys.stdin.buffer.read() - # Try to guess type - mimetype = mimetype_from_string(content) - if mimetype is None: - raise AttachmentError("Could not determine mimetype of stdin") - return Attachment(type=mimetype, path=None, url=None, content=content) - - if "://" in value: - # Confirm URL exists and try to guess type - try: - response = httpx.head(value) - response.raise_for_status() - mimetype = response.headers.get("content-type") - except httpx.HTTPError as ex: - raise AttachmentError(str(ex)) - return Attachment(type=mimetype, path=None, url=value, content=None) - - # Check that the file exists - path = pathlib.Path(value) - if not path.exists(): - raise AttachmentError(f"File {value} does not exist") - path = path.resolve() - - # Try to guess type - mimetype = mimetype_from_path(str(path)) - if mimetype is None: - raise AttachmentError(f"Could not determine mimetype of {value}") - - return Attachment(type=mimetype, path=str(path), url=None, content=None) - - -class AttachmentType(click.ParamType): - name = "attachment" - - def convert(self, value, param, ctx): - try: - return resolve_attachment(value) - except AttachmentError as e: - self.fail(str(e), param, ctx) - - -def resolve_attachment_with_type(value: str, mimetype: str) -> Attachment: - if "://" in value: - attachment = Attachment(mimetype, None, value, None) - elif value == "-": - content = sys.stdin.buffer.read() - attachment = Attachment(mimetype, None, None, content) - else: - # Look for file - path = pathlib.Path(value) - if not path.exists(): - raise click.BadParameter(f"File {value} does not exist") - path = path.resolve() - attachment = Attachment(mimetype, str(path), None, None) - return attachment - - -def attachment_types_callback(ctx, param, values) -> List[Attachment]: - collected = [] - for value, mimetype in values: - collected.append(resolve_attachment_with_type(value, mimetype)) - return collected - - -def json_validator(object_name): - def validator(ctx, param, value): - if value is None: - return value - try: - obj = json.loads(value) - if not isinstance(obj, dict): - raise click.BadParameter(f"{object_name} must be a JSON object") - return obj - except json.JSONDecodeError: - raise click.BadParameter(f"{object_name} must be valid JSON") - - return validator - - -def schema_option(fn): - click.option( - "schema_input", - "--schema", - help="JSON schema, filepath or ID", - )(fn) - return fn - - -@click.group( - cls=DefaultGroup, - default="prompt", - default_if_no_args=True, - context_settings={"help_option_names": ["-h", "--help"]}, -) -@click.version_option() -def cli(): - """ - Access Large Language Models from the command-line - - Documentation: https://llm.datasette.io/ - - LLM can run models from many different providers. Consult the - plugin directory for a list of available models: - - https://llm.datasette.io/en/stable/plugins/directory.html - - To get started with OpenAI, obtain an API key from them and: - - \b - $ llm keys set openai - Enter key: ... - - Then execute a prompt like this: - - llm 'Five outrageous names for a pet pelican' - - For a full list of prompting options run: - - llm prompt --help - """ - - -@cli.command(name="prompt") -@click.argument("prompt", required=False) -@click.option("-s", "--system", help="System prompt to use") -@click.option("model_id", "-m", "--model", help="Model to use", envvar="LLM_MODEL") -@click.option( - "-d", - "--database", - type=click.Path(readable=True, dir_okay=False), - help="Path to log database", -) -@click.option( - "queries", - "-q", - "--query", - multiple=True, - help="Use first model matching these strings", -) -@click.option( - "attachments", - "-a", - "--attachment", - type=AttachmentType(), - multiple=True, - help="Attachment path or URL or -", -) -@click.option( - "attachment_types", - "--at", - "--attachment-type", - type=(str, str), - multiple=True, - callback=attachment_types_callback, - help="\b\nAttachment with explicit mimetype,\n--at image.jpg image/jpeg", -) -@click.option( - "tools", - "-T", - "--tool", - multiple=True, - help="Name of a tool to make available to the model", -) -@click.option( - "python_tools", - "--functions", - help="Python code block or file path defining functions to register as tools", - multiple=True, -) -@click.option( - "tools_debug", - "--td", - "--tools-debug", - is_flag=True, - help="Show full details of tool executions", - envvar="LLM_TOOLS_DEBUG", -) -@click.option( - "tools_approve", - "--ta", - "--tools-approve", - is_flag=True, - help="Manually approve every tool execution", -) -@click.option( - "chain_limit", - "--cl", - "--chain-limit", - type=int, - default=5, - help="How many chained tool responses to allow, default 5, set 0 for unlimited", -) -@click.option( - "options", - "-o", - "--option", - type=(str, str), - multiple=True, - help="key/value options for the model", -) -@schema_option -@click.option( - "--schema-multi", - help="JSON schema to use for multiple results", -) -@click.option( - "fragments", - "-f", - "--fragment", - multiple=True, - help="Fragment (alias, URL, hash or file path) to add to the prompt", -) -@click.option( - "system_fragments", - "--sf", - "--system-fragment", - multiple=True, - help="Fragment to add to system prompt", -) -@click.option("-t", "--template", help="Template to use") -@click.option( - "-p", - "--param", - multiple=True, - type=(str, str), - help="Parameters for template", -) -@click.option("--no-stream", is_flag=True, help="Do not stream output") -@click.option("-n", "--no-log", is_flag=True, help="Don't log to database") -@click.option("--log", is_flag=True, help="Log prompt and response to the database") -@click.option( - "-R", "--no-reasoning", is_flag=True, help="Don't display reasoning output" -) -@click.option( - "_continue", - "-c", - "--continue", - is_flag=True, - flag_value=-1, - help="Continue the most recent conversation.", -) -@click.option( - "conversation_id", - "--cid", - "--conversation", - help="Continue the conversation with the given ID.", -) -@click.option("--key", help="API key to use") -@click.option("--save", help="Save prompt with this template name") -@click.option("async_", "--async", is_flag=True, help="Run prompt asynchronously") -@click.option("-u", "--usage", is_flag=True, help="Show token usage") -@click.option("-x", "--extract", is_flag=True, help="Extract first fenced code block") -@click.option( - "extract_last", - "--xl", - "--extract-last", - is_flag=True, - help="Extract last fenced code block", -) -def prompt( - prompt, - system, - model_id, - database, - queries, - attachments, - attachment_types, - tools, - python_tools, - tools_debug, - tools_approve, - chain_limit, - options, - schema_input, - schema_multi, - fragments, - system_fragments, - template, - param, - no_stream, - no_log, - log, - no_reasoning, - _continue, - conversation_id, - key, - save, - async_, - usage, - extract, - extract_last, -): - """ - Execute a prompt - - Documentation: https://llm.datasette.io/en/stable/usage.html - - Examples: - - \b - llm 'Capital of France?' - llm 'Capital of France?' -m gpt-4o - llm 'Capital of France?' -s 'answer in Spanish' - - Multi-modal models can be called with attachments like this: - - \b - llm 'Extract text from this image' -a image.jpg - llm 'Describe' -a https://static.simonwillison.net/static/2024/pelicans.jpg - cat image | llm 'describe image' -a - - # With an explicit mimetype: - cat image | llm 'describe image' --at - image/jpeg - - The -x/--extract option returns just the content of the first ``` fenced code - block, if one is present. If none are present it returns the full response. - - \b - llm 'JavaScript function for reversing a string' -x - """ - if log and no_log: - raise click.ClickException("--log and --no-log are mutually exclusive") - - log_path = pathlib.Path(database) if database else logs_db_path() - (log_path.parent).mkdir(parents=True, exist_ok=True) - db = sqlite_utils.Database(log_path) - migrate(db) - - if queries and not model_id: - # Use -q options to find model with shortest model_id - matches = [] - for model_with_aliases in get_models_with_aliases(): - if all(model_with_aliases.matches(q) for q in queries): - matches.append(model_with_aliases.model.model_id) - if not matches: - raise click.ClickException( - "No model found matching queries {}".format(", ".join(queries)) - ) - model_id = min(matches, key=len) - - if schema_multi: - schema_input = schema_multi - - schema = resolve_schema_input(db, schema_input, load_template) - - if schema_multi: - # Convert that schema into multiple "items" of the same schema - schema = multi_schema(schema) - - def read_prompt(): - nonlocal prompt, schema - - # Is there extra prompt available on stdin? - stdin_prompt = None - if not sys.stdin.isatty(): - stdin_prompt = sys.stdin.read() - - if stdin_prompt: - bits = [stdin_prompt] - if prompt: - bits.append(prompt) - prompt = " ".join(bits) - - if ( - prompt is None - and not save - and sys.stdin.isatty() - and not attachments - and not attachment_types - and not schema - and not fragments - ): - # Hang waiting for input to stdin (unless --save) - prompt = sys.stdin.read() - return prompt - - if save: - # We are saving their prompt/system/etc to a new template - # Fields to save: prompt, system, model - and more in the future - disallowed_options = [] - for option, var in ( - ("--template", template), - ("--continue", _continue), - ("--cid", conversation_id), - ): - if var: - disallowed_options.append(option) - if disallowed_options: - raise click.ClickException( - "--save cannot be used with {}".format(", ".join(disallowed_options)) - ) - path = template_dir() / f"{save}.yaml" - to_save = {} - if model_id: - model_aliases = get_model_aliases() - try: - to_save["model"] = model_aliases[model_id].model_id - except KeyError: - raise click.ClickException("'{}' is not a known model".format(model_id)) - prompt = read_prompt() - if prompt: - to_save["prompt"] = prompt - if system: - to_save["system"] = system - if param: - to_save["defaults"] = dict(param) - if extract: - to_save["extract"] = True - if extract_last: - to_save["extract_last"] = True - if schema: - to_save["schema_object"] = schema - if fragments: - to_save["fragments"] = list(fragments) - if system_fragments: - to_save["system_fragments"] = list(system_fragments) - if python_tools: - to_save["functions"] = "\n\n".join(python_tools) - if tools: - to_save["tools"] = list(tools) - if attachments: - # Only works for attachments with a path or url - to_save["attachments"] = [ - (a.path or a.url) for a in attachments if (a.path or a.url) - ] - if attachment_types: - to_save["attachment_types"] = [ - {"type": a.type, "value": a.path or a.url} - for a in attachment_types - if (a.path or a.url) - ] - if options: - # Need to validate and convert their types first - model = get_model(model_id or get_default_model()) - try: - options_model = model.Options(**dict(options)) - # Use model_dump(mode="json") so Enums become their .value strings - to_save["options"] = { - k: v - for k, v in options_model.model_dump(mode="json").items() - if v is not None - } - except pydantic.ValidationError as ex: - raise click.ClickException(render_errors(ex.errors())) - path.write_text( - yaml.safe_dump( - to_save, - indent=4, - default_flow_style=False, - sort_keys=False, - ), - "utf-8", - ) - return - - if template: - params = dict(param) - # Cannot be used with system - try: - template_obj = load_template(template) - except LoadTemplateError as ex: - raise click.ClickException(str(ex)) - if not (extract or extract_last): - extract = template_obj.extract - extract_last = template_obj.extract_last - # Combine with template fragments/system_fragments - if template_obj.fragments: - fragments = [*template_obj.fragments, *fragments] - if template_obj.system_fragments: - system_fragments = [*template_obj.system_fragments, *system_fragments] - if template_obj.schema_object: - schema = template_obj.schema_object - if template_obj.tools: - tools = [*template_obj.tools, *tools] - if template_obj.functions and template_obj._functions_is_trusted: - python_tools = [template_obj.functions, *python_tools] - input_ = "" - if template_obj.options: - # Make options mutable (they start as a tuple) - options = list(options) - # Load any options, provided they were not set using -o already - specified_options = dict(options) - for option_name, option_value in template_obj.options.items(): - if option_name not in specified_options: - options.append((option_name, option_value)) - if "input" in template_obj.vars(): - input_ = read_prompt() - try: - template_prompt, template_system = template_obj.evaluate(input_, params) - if template_prompt: - # Combine with user prompt - if prompt and "input" not in template_obj.vars(): - prompt = template_prompt + "\n" + prompt - else: - prompt = template_prompt - if template_system and not system: - system = template_system - except Template.MissingVariables as ex: - raise click.ClickException(str(ex)) - if model_id is None and template_obj.model: - model_id = template_obj.model - # Merge in any attachments - if template_obj.attachments: - attachments = [ - resolve_attachment(a) for a in template_obj.attachments - ] + list(attachments) - if template_obj.attachment_types: - attachment_types = [ - resolve_attachment_with_type(at.value, at.type) - for at in template_obj.attachment_types - ] + list(attachment_types) - if extract or extract_last: - no_stream = True - - conversation = None - if conversation_id or _continue: - # Load the conversation - loads most recent if no ID provided - try: - conversation = load_conversation( - conversation_id, async_=async_, database=database - ) - except UnknownModelError as ex: - raise click.ClickException(str(ex)) - - if conversation_tools := _get_conversation_tools(conversation, tools): - tools = conversation_tools - - # Figure out which model we are using - if model_id is None: - if conversation: - model_id = conversation.model.model_id - else: - model_id = get_default_model() - - # Now resolve the model - try: - if async_: - model = get_async_model(model_id) - else: - model = get_model(model_id) - except UnknownModelError as ex: - raise click.ClickException(ex) - - if conversation is None and (tools or python_tools): - conversation = model.conversation() - - if conversation: - # To ensure it can see the key - conversation.model = model - - # Validate options - validated_options = {} - if options: - # Validate with pydantic - try: - validated_options = dict( - (key, value) - for key, value in model.Options(**dict(options)) - if value is not None - ) - except pydantic.ValidationError as ex: - raise click.ClickException(render_errors(ex.errors())) - - # Add on any default model options - default_options = get_model_options(model.model_id) - for key_, value in default_options.items(): - if key_ not in validated_options: - validated_options[key_] = value - - kwargs = {} - - resolved_attachments = [*attachments, *attachment_types] - - should_stream = model.can_stream and not no_stream - if not should_stream: - kwargs["stream"] = False - - if isinstance(model, (KeyModel, AsyncKeyModel)): - kwargs["key"] = key - - prompt = read_prompt() - response = None - - try: - fragments_and_attachments = resolve_fragments( - db, fragments, allow_attachments=True - ) - resolved_fragments = [ - fragment - for fragment in fragments_and_attachments - if isinstance(fragment, Fragment) - ] - resolved_attachments.extend( - attachment - for attachment in fragments_and_attachments - if isinstance(attachment, Attachment) - ) - resolved_system_fragments = resolve_fragments(db, system_fragments) - except FragmentNotFound as ex: - raise click.ClickException(str(ex)) - - prompt_method = model.prompt - if conversation: - prompt_method = conversation.prompt - - tool_implementations = _gather_tools(tools, python_tools) - - if tool_implementations: - prompt_method = conversation.chain - kwargs["options"] = validated_options - kwargs["chain_limit"] = chain_limit - if tools_debug: - kwargs["after_call"] = _debug_tool_call - if tools_approve: - kwargs["before_call"] = _approve_tool_call - kwargs["tools"] = tool_implementations - else: - # Merge in options for the .prompt() methods - kwargs.update(validated_options) - - try: - if async_: - - async def inner(): - if should_stream: - response = prompt_method( - prompt, - attachments=resolved_attachments, - system=system, - schema=schema, - fragments=resolved_fragments, - system_fragments=resolved_system_fragments, - **kwargs, - ) - await display_async_stream_events( - response.astream_events(), - show_reasoning=not no_reasoning, - ) - print("") - else: - response = prompt_method( - prompt, - fragments=resolved_fragments, - attachments=resolved_attachments, - schema=schema, - system=system, - system_fragments=resolved_system_fragments, - **kwargs, - ) - text = await response.text() - if extract or extract_last: - text = ( - extract_fenced_code_block(text, last=extract_last) or text - ) - print(text) - return response - - response = asyncio.run(inner()) - else: - response = prompt_method( - prompt, - fragments=resolved_fragments, - attachments=resolved_attachments, - system=system, - schema=schema, - system_fragments=resolved_system_fragments, - **kwargs, - ) - if should_stream: - display_stream_events( - response.stream_events(), - show_reasoning=not no_reasoning, - ) - print("") - else: - text = response.text() - if extract or extract_last: - text = extract_fenced_code_block(text, last=extract_last) or text - print(text) - # List of exceptions that should never be raised in pytest: - except (ValueError, NotImplementedError) as ex: - raise click.ClickException(str(ex)) - except Exception as ex: - # All other exceptions should raise in pytest, show to user otherwise - if getattr(sys, "_called_from_test", False) or os.environ.get( - "LLM_RAISE_ERRORS", None - ): - raise - raise click.ClickException(str(ex)) - - if usage: - if isinstance(response, ChainResponse): - responses = response._responses - else: - responses = [response] - for response_object in responses: - # Show token usage to stderr in yellow - click.echo( - click.style( - "Token usage: {}".format(response_object.token_usage()), - fg="yellow", - bold=True, - ), - err=True, - ) - - # Log responses to the database - if (logs_on() or log) and not no_log: - # Could be Response, AsyncResponse, ChainResponse, AsyncChainResponse - if isinstance(response, AsyncResponse): - response = asyncio.run(response.to_sync_response()) - # At this point ALL forms should have a log_to_db() method that works: - response.log_to_db(db) - - -@cli.command() -@click.option("-s", "--system", help="System prompt to use") -@click.option("model_id", "-m", "--model", help="Model to use", envvar="LLM_MODEL") -@click.option( - "_continue", - "-c", - "--continue", - is_flag=True, - flag_value=-1, - help="Continue the most recent conversation.", -) -@click.option( - "conversation_id", - "--cid", - "--conversation", - help="Continue the conversation with the given ID.", -) -@click.option( - "fragments", - "-f", - "--fragment", - multiple=True, - help="Fragment (alias, URL, hash or file path) to add to the prompt", -) -@click.option( - "system_fragments", - "--sf", - "--system-fragment", - multiple=True, - help="Fragment to add to system prompt", -) -@click.option("-t", "--template", help="Template to use") -@click.option( - "-p", - "--param", - multiple=True, - type=(str, str), - help="Parameters for template", -) -@click.option( - "options", - "-o", - "--option", - type=(str, str), - multiple=True, - help="key/value options for the model", -) -@click.option( - "-d", - "--database", - type=click.Path(readable=True, dir_okay=False), - help="Path to log database", -) -@click.option("--no-stream", is_flag=True, help="Do not stream output") -@click.option( - "-R", "--no-reasoning", is_flag=True, help="Don't display reasoning output" -) -@click.option("--key", help="API key to use") -@click.option( - "tools", - "-T", - "--tool", - multiple=True, - help="Name of a tool to make available to the model", -) -@click.option( - "python_tools", - "--functions", - help="Python code block or file path defining functions to register as tools", - multiple=True, -) -@click.option( - "tools_debug", - "--td", - "--tools-debug", - is_flag=True, - help="Show full details of tool executions", - envvar="LLM_TOOLS_DEBUG", -) -@click.option( - "tools_approve", - "--ta", - "--tools-approve", - is_flag=True, - help="Manually approve every tool execution", -) -@click.option( - "chain_limit", - "--cl", - "--chain-limit", - type=int, - default=5, - help="How many chained tool responses to allow, default 5, set 0 for unlimited", -) -def chat( - system, - model_id, - _continue, - conversation_id, - fragments, - system_fragments, - template, - param, - options, - no_stream, - no_reasoning, - key, - database, - tools, - python_tools, - tools_debug, - tools_approve, - chain_limit, -): - """ - Hold an ongoing chat with a model. - """ - # Left and right arrow keys to move cursor: - if sys.platform != "win32": - readline.parse_and_bind("\\e[D: backward-char") - readline.parse_and_bind("\\e[C: forward-char") - else: - readline.parse_and_bind("bind -x '\\e[D: backward-char'") - readline.parse_and_bind("bind -x '\\e[C: forward-char'") - log_path = pathlib.Path(database) if database else logs_db_path() - (log_path.parent).mkdir(parents=True, exist_ok=True) - db = sqlite_utils.Database(log_path) - migrate(db) - - conversation = None - if conversation_id or _continue: - # Load the conversation - loads most recent if no ID provided - try: - conversation = load_conversation(conversation_id, database=database) - except UnknownModelError as ex: - raise click.ClickException(str(ex)) - - if conversation_tools := _get_conversation_tools(conversation, tools): - tools = conversation_tools - - template_obj = None - if template: - params = dict(param) - try: - template_obj = load_template(template) - except LoadTemplateError as ex: - raise click.ClickException(str(ex)) - if model_id is None and template_obj.model: - model_id = template_obj.model - if template_obj.tools: - tools = [*template_obj.tools, *tools] - if template_obj.functions and template_obj._functions_is_trusted: - python_tools = [template_obj.functions, *python_tools] - - # Figure out which model we are using - if model_id is None: - if conversation: - model_id = conversation.model.model_id - else: - model_id = get_default_model() - - # Now resolve the model - try: - model = get_model(model_id) - except KeyError: - raise click.ClickException("'{}' is not a known model".format(model_id)) - - if conversation is None: - # Start a fresh conversation for this chat - conversation = Conversation(model=model) - else: - # Ensure it can see the API key - conversation.model = model - - if tools_debug: - conversation.after_call = _debug_tool_call - if tools_approve: - conversation.before_call = _approve_tool_call - - # Validate options - validated_options = get_model_options(model.model_id) - if options: - try: - validated_options = dict( - (key, value) - for key, value in model.Options(**dict(options)) - if value is not None - ) - except pydantic.ValidationError as ex: - raise click.ClickException(render_errors(ex.errors())) - - kwargs = {} - if validated_options: - kwargs["options"] = validated_options - - tool_functions = _gather_tools(tools, python_tools) - - if tool_functions: - kwargs["chain_limit"] = chain_limit - kwargs["tools"] = tool_functions - - should_stream = model.can_stream and not no_stream - if not should_stream: - kwargs["stream"] = False - - if key and isinstance(model, KeyModel): - kwargs["key"] = key - - try: - fragments_and_attachments = resolve_fragments( - db, fragments, allow_attachments=True - ) - argument_fragments = [ - fragment - for fragment in fragments_and_attachments - if isinstance(fragment, Fragment) - ] - argument_attachments = [ - attachment - for attachment in fragments_and_attachments - if isinstance(attachment, Attachment) - ] - argument_system_fragments = resolve_fragments(db, system_fragments) - except FragmentNotFound as ex: - raise click.ClickException(str(ex)) - - click.echo("Chatting with {}".format(model.model_id)) - click.echo("Type 'exit' or 'quit' to exit") - click.echo("Type '!multi' to enter multiple lines, then '!end' to finish") - click.echo("Type '!edit' to open your default editor and modify the prompt") - click.echo( - "Type '!fragment [ ...]' to insert one or more fragments" - ) - in_multi = False - - accumulated = [] - accumulated_fragments = [] - accumulated_attachments = [] - end_token = "!end" - while True: - prompt = click.prompt("", prompt_suffix="> " if not in_multi else "") - fragments = [] - attachments = [] - if argument_fragments: - fragments += argument_fragments - # fragments from --fragments will get added to the first message only - argument_fragments = [] - if argument_attachments: - attachments = argument_attachments - argument_attachments = [] - if prompt.strip().startswith("!multi"): - in_multi = True - bits = prompt.strip().split() - if len(bits) > 1: - end_token = "!end {}".format(" ".join(bits[1:])) - continue - if prompt.strip() == "!edit": - edited_prompt = click.edit() - if edited_prompt is None: - click.echo("Editor closed without saving.", err=True) - continue - prompt = edited_prompt.strip() - if prompt.strip().startswith("!fragment "): - prompt, fragments, attachments = process_fragments_in_chat(db, prompt) - - if in_multi: - if prompt.strip() == end_token: - prompt = "\n".join(accumulated) - fragments = accumulated_fragments - attachments = accumulated_attachments - in_multi = False - accumulated = [] - accumulated_fragments = [] - accumulated_attachments = [] - else: - if prompt: - accumulated.append(prompt) - accumulated_fragments += fragments - accumulated_attachments += attachments - continue - if template_obj: - try: - # Mirror prompt() logic: only pass input if template uses it - uses_input = "input" in template_obj.vars() - input_ = prompt if uses_input else "" - template_prompt, template_system = template_obj.evaluate(input_, params) - except Template.MissingVariables as ex: - raise click.ClickException(str(ex)) - if template_system and not system: - system = template_system - if template_prompt: - if prompt and not uses_input: - prompt = f"{template_prompt}\n{prompt}" - else: - prompt = template_prompt - if prompt.strip() in ("exit", "quit"): - break - - response = conversation.chain( - prompt, - fragments=fragments, - system_fragments=argument_system_fragments, - attachments=attachments, - system=system, - **kwargs, - ) - - # System prompt and system fragments only sent for the first message - system = None - argument_system_fragments = [] - display_stream_events( - response.stream_events(), - show_reasoning=not no_reasoning, - ) - response.log_to_db(db) - print("") - - -def load_conversation( - conversation_id: Optional[str], - async_=False, - database=None, -) -> Optional[_BaseConversation]: - log_path = pathlib.Path(database) if database else logs_db_path() - db = sqlite_utils.Database(log_path) - migrate(db) - if conversation_id is None: - # Return the most recent conversation, or None if there are none - matches = list(db["conversations"].rows_where(order_by="id desc", limit=1)) - if matches: - conversation_id = matches[0]["id"] - else: - return None - try: - row = cast(sqlite_utils.db.Table, db["conversations"]).get(conversation_id) - except sqlite_utils.db.NotFoundError: - raise click.ClickException( - "No conversation found with id={}".format(conversation_id) - ) - # Inflate that conversation - conversation_class = AsyncConversation if async_ else Conversation - response_class = AsyncResponse if async_ else Response - conversation = conversation_class.from_row(row) - for response in db["responses"].rows_where( - "conversation_id = ?", [conversation_id] - ): - conversation.responses.append(response_class.from_row(db, response)) - return conversation - - -@cli.group( - cls=DefaultGroup, - default="list", - default_if_no_args=True, -) -def keys(): - "Manage stored API keys for different models" - - -@keys.command(name="list") -def keys_list(): - "List names of all stored keys" - path = user_dir() / "keys.json" - if not path.exists(): - click.echo("No keys found") - return - keys = json.loads(path.read_text()) - for key in sorted(keys.keys()): - if key != "// Note": - click.echo(key) - - -@keys.command(name="path") -def keys_path_command(): - "Output the path to the keys.json file" - click.echo(user_dir() / "keys.json") - - -@keys.command(name="get") -@click.argument("name") -def keys_get(name): - """ - Return the value of a stored key - - Example usage: - - \b - export OPENAI_API_KEY=$(llm keys get openai) - """ - path = user_dir() / "keys.json" - if not path.exists(): - raise click.ClickException("No keys found") - keys = json.loads(path.read_text()) - try: - click.echo(keys[name]) - except KeyError: - raise click.ClickException("No key found with name '{}'".format(name)) - - -@keys.command(name="set") -@click.argument("name") -@click.option("--value", prompt="Enter key", hide_input=True, help="Value to set") -def keys_set(name, value): - """ - Save a key in the keys.json file - - Example usage: - - \b - $ llm keys set openai - Enter key: ... - """ - default = {"// Note": "This file stores secret API credentials. Do not share!"} - path = user_dir() / "keys.json" - path.parent.mkdir(parents=True, exist_ok=True) - if not path.exists(): - path.write_text(json.dumps(default)) - path.chmod(0o600) - try: - current = json.loads(path.read_text()) - except json.decoder.JSONDecodeError: - current = default - current[name] = value - path.write_text(json.dumps(current, indent=2) + "\n") - - -@cli.group( - cls=DefaultGroup, - default="list", - default_if_no_args=True, -) -def logs(): - "Tools for exploring logged prompts and responses" - - -@logs.command(name="path") -def logs_path(): - "Output the path to the logs.db file" - click.echo(logs_db_path()) - - -@logs.command(name="status") -def logs_status(): - "Show current status of database logging" - path = logs_db_path() - if not path.exists(): - click.echo("No log database found at {}".format(path)) - return - if logs_on(): - click.echo("Logging is ON for all prompts".format()) - else: - click.echo("Logging is OFF".format()) - db = sqlite_utils.Database(path) - migrate(db) - click.echo("Found log database at {}".format(path)) - click.echo("Number of conversations logged:\t{}".format(db["conversations"].count)) - click.echo("Number of responses logged:\t{}".format(db["responses"].count)) - click.echo( - "Database file size: \t\t{}".format(_human_readable_size(path.stat().st_size)) - ) - - -@logs.command(name="backup") -@click.argument("path", type=click.Path(dir_okay=True, writable=True)) -def backup(path): - "Backup your logs database to this file" - logs_path = logs_db_path() - path = pathlib.Path(path) - db = sqlite_utils.Database(logs_path) - try: - db.execute("vacuum into ?", [str(path)]) - except Exception as ex: - raise click.ClickException(str(ex)) - click.echo( - "Backed up {} to {}".format(_human_readable_size(path.stat().st_size), path) - ) - - -@logs.command(name="on") -def logs_turn_on(): - "Turn on logging for all prompts" - path = user_dir() / "logs-off" - if path.exists(): - path.unlink() - - -@logs.command(name="off") -def logs_turn_off(): - "Turn off logging for all prompts" - path = user_dir() / "logs-off" - path.touch() - - -LOGS_COLUMNS = """ responses.id, - responses.model, - responses.resolved_model, - responses.prompt, - responses.system, - responses.prompt_json, - responses.options_json, - responses.response, - responses.response_json, - responses.conversation_id, - responses.duration_ms, - responses.datetime_utc, - responses.input_tokens, - responses.output_tokens, - responses.token_details, - conversations.name as conversation_name, - conversations.model as conversation_model, - schemas.content as schema_json""" - -LOGS_SQL = """ -select -{columns} -from - responses -left join schemas on responses.schema_id = schemas.id -left join conversations on responses.conversation_id = conversations.id{extra_where} -order by {order_by}{limit} -""" -LOGS_SQL_SEARCH = """ -select -{columns} -from - responses -left join schemas on responses.schema_id = schemas.id -left join conversations on responses.conversation_id = conversations.id -join responses_fts on responses_fts.rowid = responses.rowid -where responses_fts match :query{extra_where} -order by {order_by}{limit} -""" - -ATTACHMENTS_SQL = """ -select - response_id, - attachments.id, - attachments.type, - attachments.path, - attachments.url, - length(attachments.content) as content_length -from attachments -join prompt_attachments - on attachments.id = prompt_attachments.attachment_id -where prompt_attachments.response_id in ({}) -order by prompt_attachments."order" -""" - - -@logs.command(name="list") -@click.option( - "-n", - "--count", - type=int, - default=None, - help="Number of entries to show - defaults to 3, use 0 for all", -) -@click.option( - "-p", - "--path", - type=click.Path(readable=True, exists=True, dir_okay=False), - help="Path to log database", - hidden=True, -) -@click.option( - "-d", - "--database", - type=click.Path(readable=True, exists=True, dir_okay=False), - help="Path to log database", -) -@click.option("-m", "--model", help="Filter by model or model alias") -@click.option("-q", "--query", help="Search for logs matching this string") -@click.option( - "fragments", - "--fragment", - "-f", - help="Filter for prompts using these fragments", - multiple=True, -) -@click.option( - "tools", - "-T", - "--tool", - multiple=True, - help="Filter for prompts with results from these tools", -) -@click.option( - "any_tools", - "--tools", - is_flag=True, - help="Filter for prompts with results from any tools", -) -@schema_option -@click.option( - "--schema-multi", - help="JSON schema used for multiple results", -) -@click.option( - "-l", "--latest", is_flag=True, help="Return latest results matching search query" -) -@click.option( - "--data", is_flag=True, help="Output newline-delimited JSON data for schema" -) -@click.option("--data-array", is_flag=True, help="Output JSON array of data for schema") -@click.option("--data-key", help="Return JSON objects from array in this key") -@click.option( - "--data-ids", is_flag=True, help="Attach corresponding IDs to JSON objects" -) -@click.option("-t", "--truncate", is_flag=True, help="Truncate long strings in output") -@click.option( - "-s", "--short", is_flag=True, help="Shorter YAML output with truncated prompts" -) -@click.option("-u", "--usage", is_flag=True, help="Include token usage") -@click.option("-r", "--response", is_flag=True, help="Just output the last response") -@click.option("-x", "--extract", is_flag=True, help="Extract first fenced code block") -@click.option( - "extract_last", - "--xl", - "--extract-last", - is_flag=True, - help="Extract last fenced code block", -) -@click.option( - "current_conversation", - "-c", - "--current", - is_flag=True, - flag_value=-1, - help="Show logs from the current conversation", -) -@click.option( - "conversation_id", - "--cid", - "--conversation", - help="Show logs for this conversation ID", -) -@click.option("--id-gt", help="Return responses with ID > this") -@click.option("--id-gte", help="Return responses with ID >= this") -@click.option( - "json_output", - "--json", - is_flag=True, - help="Output logs as JSON", -) -@click.option( - "--expand", - "-e", - is_flag=True, - help="Expand fragments to show their content", -) -def logs_list( - count, - path, - database, - model, - query, - fragments, - tools, - any_tools, - schema_input, - schema_multi, - latest, - data, - data_array, - data_key, - data_ids, - truncate, - short, - usage, - response, - extract, - extract_last, - current_conversation, - conversation_id, - id_gt, - id_gte, - json_output, - expand, -): - "Show logged prompts and their responses" - if database and not path: - path = database - path = pathlib.Path(path or logs_db_path()) - if not path.exists(): - raise click.ClickException("No log database found at {}".format(path)) - db = sqlite_utils.Database(path) - migrate(db) - - if schema_multi: - schema_input = schema_multi - schema = resolve_schema_input(db, schema_input, load_template) - if schema_multi: - schema = multi_schema(schema) - - if short and (json_output or response): - invalid = " or ".join( - [ - flag[0] - for flag in (("--json", json_output), ("--response", response)) - if flag[1] - ] - ) - raise click.ClickException("Cannot use --short and {} together".format(invalid)) - - if response and not current_conversation and not conversation_id: - current_conversation = True - - if current_conversation: - try: - conversation_id = next( - db.query( - "select conversation_id from responses order by id desc limit 1" - ) - )["conversation_id"] - except StopIteration: - # No conversations yet - raise click.ClickException("No conversations found") - - # For --conversation set limit 0, if not explicitly set - if count is None: - if conversation_id: - count = 0 - else: - count = 3 - - model_id = None - if model: - # Resolve alias, if any - try: - model_id = get_model(model).model_id - except UnknownModelError: - # Maybe they uninstalled a model, use the -m option as-is - model_id = model - - sql = LOGS_SQL - order_by = "responses.id desc" - if query: - sql = LOGS_SQL_SEARCH - if not latest: - order_by = "responses_fts.rank desc" - - limit = "" - if count is not None and count > 0: - limit = " limit {}".format(count) - - sql_format = { - "limit": limit, - "columns": LOGS_COLUMNS, - "extra_where": "", - "order_by": order_by, - } - where_bits = [] - sql_params = { - "model": model_id, - "query": query, - "conversation_id": conversation_id, - "id_gt": id_gt, - "id_gte": id_gte, - } - if model_id: - where_bits.append("responses.model = :model") - if conversation_id: - where_bits.append("responses.conversation_id = :conversation_id") - if id_gt: - where_bits.append("responses.id > :id_gt") - if id_gte: - where_bits.append("responses.id >= :id_gte") - if fragments: - # Resolve the fragments to their hashes - fragment_hashes = [ - fragment.id() for fragment in resolve_fragments(db, fragments) - ] - exists_clauses = [] - - for i, fragment_hash in enumerate(fragment_hashes): - exists_clause = f""" - exists ( - select 1 from prompt_fragments - where prompt_fragments.response_id = responses.id - and prompt_fragments.fragment_id in ( - select fragments.id from fragments - where hash = :f{i} - ) - union - select 1 from system_fragments - where system_fragments.response_id = responses.id - and system_fragments.fragment_id in ( - select fragments.id from fragments - where hash = :f{i} - ) - ) - """ - exists_clauses.append(exists_clause) - sql_params["f{}".format(i)] = fragment_hash - - where_bits.append(" and ".join(exists_clauses)) - - if any_tools: - # Any response that involved at least one tool result - where_bits.append(""" - exists ( - select 1 - from tool_results - where - tool_results.response_id = responses.id - ) - """) - if tools: - tools_by_name = get_tools() - # Filter responses by tools (must have ALL of the named tools, including plugin) - tool_clauses = [] - for i, tool_name in enumerate(tools): - try: - plugin_name = tools_by_name[tool_name].plugin - except KeyError: - raise click.ClickException(f"Unknown tool: {tool_name}") - - tool_clauses.append(f""" - exists ( - select 1 - from tool_results - join tools on tools.id = tool_results.tool_id - where tool_results.response_id = responses.id - and tools.name = :tool{i} - and tools.plugin = :plugin{i} - ) - """) - sql_params[f"tool{i}"] = tool_name - sql_params[f"plugin{i}"] = plugin_name - - # AND means “must have all” — use OR instead if you want “any of” - where_bits.append(" and ".join(tool_clauses)) - - schema_id = None - if schema: - schema_id = make_schema_id(schema)[0] - where_bits.append("responses.schema_id = :schema_id") - sql_params["schema_id"] = schema_id - - if where_bits: - where_ = " and " if query else " where " - sql_format["extra_where"] = where_ + " and ".join(where_bits) - - final_sql = sql.format(**sql_format) - rows = list(db.query(final_sql, sql_params)) - - # Reverse the order - we do this because we 'order by id desc limit 3' to get the - # 3 most recent results, but we still want to display them in chronological order - # ... except for searches where we don't do this - if not query and not data: - rows.reverse() - - # Fetch any attachments - ids = [row["id"] for row in rows] - attachments = list(db.query(ATTACHMENTS_SQL.format(",".join("?" * len(ids))), ids)) - attachments_by_id = {} - for attachment in attachments: - attachments_by_id.setdefault(attachment["response_id"], []).append(attachment) - - FRAGMENTS_SQL = """ - select - {table}.response_id, - fragments.hash, - fragments.id as fragment_id, - fragments.content, - ( - select json_group_array(fragment_aliases.alias) - from fragment_aliases - where fragment_aliases.fragment_id = fragments.id - ) as aliases - from {table} - join fragments on {table}.fragment_id = fragments.id - where {table}.response_id in ({placeholders}) - order by {table}."order" - """ - - # Fetch any prompt or system prompt fragments - prompt_fragments_by_id = {} - system_fragments_by_id = {} - for table, dictionary in ( - ("prompt_fragments", prompt_fragments_by_id), - ("system_fragments", system_fragments_by_id), - ): - for fragment in db.query( - FRAGMENTS_SQL.format(placeholders=",".join("?" * len(ids)), table=table), - ids, - ): - dictionary.setdefault(fragment["response_id"], []).append(fragment) - - if data or data_array or data_key or data_ids: - # Special case for --data to output valid JSON - to_output = [] - for row in rows: - response = row["response"] or "" - try: - decoded = json.loads(response) - new_items = [] - if ( - isinstance(decoded, dict) - and (data_key in decoded) - and all(isinstance(item, dict) for item in decoded[data_key]) - ): - for item in decoded[data_key]: - new_items.append(item) - else: - new_items.append(decoded) - if data_ids: - for item in new_items: - item[find_unused_key(item, "response_id")] = row["id"] - item[find_unused_key(item, "conversation_id")] = row["id"] - to_output.extend(new_items) - except ValueError: - pass - for line in output_rows_as_json(to_output, nl=not data_array, compact=True): - click.echo(line) - return - - # Tool usage information - TOOLS_SQL = """ - SELECT responses.id, - -- Tools related to this response - COALESCE( - (SELECT json_group_array(json_object( - 'id', t.id, - 'hash', t.hash, - 'name', t.name, - 'description', t.description, - 'input_schema', json(t.input_schema) - )) - FROM tools t - JOIN tool_responses tr ON t.id = tr.tool_id - WHERE tr.response_id = responses.id - ), - '[]' - ) AS tools, - -- Tool calls for this response - COALESCE( - (SELECT json_group_array(json_object( - 'id', tc.id, - 'tool_id', tc.tool_id, - 'name', tc.name, - 'arguments', json(tc.arguments), - 'tool_call_id', tc.tool_call_id - )) - FROM tool_calls tc - WHERE tc.response_id = responses.id - ), - '[]' - ) AS tool_calls, - -- Tool results for this response - COALESCE( - (SELECT json_group_array(json_object( - 'id', tr.id, - 'tool_id', tr.tool_id, - 'name', tr.name, - 'output', tr.output, - 'tool_call_id', tr.tool_call_id, - 'exception', tr.exception, - 'attachments', COALESCE( - (SELECT json_group_array(json_object( - 'id', a.id, - 'type', a.type, - 'path', a.path, - 'url', a.url, - 'content', a.content - )) - FROM tool_results_attachments tra - JOIN attachments a ON tra.attachment_id = a.id - WHERE tra.tool_result_id = tr.id - ), - '[]' - ) - )) - FROM tool_results tr - WHERE tr.response_id = responses.id - ), - '[]' - ) AS tool_results - FROM responses - where id in ({placeholders}) - """ - tool_info_by_id = { - row["id"]: { - "tools": json.loads(row["tools"]), - "tool_calls": json.loads(row["tool_calls"]), - "tool_results": json.loads(row["tool_results"]), - } - for row in db.query( - TOOLS_SQL.format(placeholders=",".join("?" * len(ids))), ids - ) - } - - for row in rows: - if truncate: - row["prompt"] = truncate_string(row["prompt"] or "") - row["response"] = truncate_string(row["response"] or "") - # Add prompt and system fragments - for key in ("prompt_fragments", "system_fragments"): - row[key] = [ - { - "hash": fragment["hash"], - "content": ( - fragment["content"] - if expand - else truncate_string(fragment["content"]) - ), - "aliases": json.loads(fragment["aliases"]), - } - for fragment in ( - prompt_fragments_by_id.get(row["id"], []) - if key == "prompt_fragments" - else system_fragments_by_id.get(row["id"], []) - ) - ] - # Either decode or remove all JSON keys - keys = list(row.keys()) - for key in keys: - if key.endswith("_json") and row[key] is not None: - if truncate: - del row[key] - else: - row[key] = json.loads(row[key]) - row.update(tool_info_by_id[row["id"]]) - - output = None - if json_output: - # Output as JSON if requested - for row in rows: - row["attachments"] = [ - {k: v for k, v in attachment.items() if k != "response_id"} - for attachment in attachments_by_id.get(row["id"], []) - ] - output = json.dumps(list(rows), indent=2) - elif extract or extract_last: - # Extract and return first code block - for row in rows: - output = extract_fenced_code_block(row["response"], last=extract_last) - if output is not None: - break - elif response: - # Just output the last response - if rows: - output = rows[-1]["response"] - - if output is not None: - click.echo(output) - else: - # Output neatly formatted human-readable logs - def _display_fragments(fragments, title): - if not fragments: - return - if not expand: - content = "\n".join( - ["- {}".format(fragment["hash"]) for fragment in fragments] - ) - else: - #
for each one - bits = [] - for fragment in fragments: - bits.append( - "
{}\n{}\n
".format( - fragment["hash"], maybe_fenced_code(fragment["content"]) - ) - ) - content = "\n".join(bits) - click.echo(f"\n### {title}\n\n{content}") - - current_system = None - should_show_conversation = True - for row in rows: - if short: - system = truncate_string( - row["system"] or "", 120, normalize_whitespace=True - ) - prompt = truncate_string( - row["prompt"] or "", 120, normalize_whitespace=True, keep_end=True - ) - cid = row["conversation_id"] - attachments = attachments_by_id.get(row["id"]) - obj = { - "model": row["model"], - "datetime": row["datetime_utc"].split(".")[0], - "conversation": cid, - } - if row["tool_calls"]: - obj["tool_calls"] = [ - "{}({})".format( - tool_call["name"], json.dumps(tool_call["arguments"]) - ) - for tool_call in row["tool_calls"] - ] - if row["tool_results"]: - obj["tool_results"] = [ - "{}: {}".format( - tool_result["name"], truncate_string(tool_result["output"]) - ) - for tool_result in row["tool_results"] - ] - if system: - obj["system"] = system - if prompt: - obj["prompt"] = prompt - if attachments: - items = [] - for attachment in attachments: - details = {"type": attachment["type"]} - if attachment.get("path"): - details["path"] = attachment["path"] - if attachment.get("url"): - details["url"] = attachment["url"] - items.append(details) - obj["attachments"] = items - for key in ("prompt_fragments", "system_fragments"): - obj[key] = [fragment["hash"] for fragment in row[key]] - if usage and (row["input_tokens"] or row["output_tokens"]): - usage_details = { - "input": row["input_tokens"], - "output": row["output_tokens"], - } - if row["token_details"]: - usage_details["details"] = json.loads(row["token_details"]) - obj["usage"] = usage_details - click.echo(yaml.dump([obj], sort_keys=False).strip()) - continue - # Not short, output Markdown - click.echo( - "# {}{}\n{}".format( - row["datetime_utc"].split(".")[0], - ( - " conversation: {} id: {}".format( - row["conversation_id"], row["id"] - ) - if should_show_conversation - else "" - ), - ( - ( - "\nModel: **{}**{}\n".format( - row["model"], - ( - " (resolved: **{}**)".format(row["resolved_model"]) - if row["resolved_model"] - else "" - ), - ) - ) - if should_show_conversation - else "" - ), - ) - ) - # In conversation log mode only show it for the first one - if conversation_id: - should_show_conversation = False - click.echo("## Prompt\n\n{}".format(row["prompt"] or "-- none --")) - _display_fragments(row["prompt_fragments"], "Prompt fragments") - if row["options_json"]: - options = row["options_json"] - if isinstance(options, str): - options = json.loads(options) - if options: - options_text = "\n".join( - "- {}: {}".format(key, value) for key, value in options.items() - ) - click.echo("\n## Options\n\n{}".format(options_text)) - if row["system"] != current_system: - if row["system"] is not None: - click.echo("\n## System\n\n{}".format(row["system"])) - current_system = row["system"] - _display_fragments(row["system_fragments"], "System fragments") - if row["schema_json"]: - click.echo( - "\n## Schema\n\n```json\n{}\n```".format( - json.dumps(row["schema_json"], indent=2) - ) - ) - # Show tool calls and results - if row["tools"]: - click.echo("\n### Tools\n") - for tool in row["tools"]: - click.echo( - "- **{}**: `{}`
\n {}
\n Arguments: {}".format( - tool["name"], - tool["hash"], - tool["description"], - json.dumps(tool["input_schema"]["properties"]), - ) - ) - if row["tool_results"]: - click.echo("\n### Tool results\n") - for tool_result in row["tool_results"]: - attachments = "" - for attachment in tool_result["attachments"]: - desc = "" - if attachment.get("type"): - desc += attachment["type"] + ": " - if attachment.get("path"): - desc += attachment["path"] - elif attachment.get("url"): - desc += attachment["url"] - elif attachment.get("content"): - desc += f"<{attachment['content_length']:,} bytes>" - attachments += "\n - {}".format(desc) - click.echo( - "- **{}**: `{}`
\n{}{}{}".format( - tool_result["name"], - tool_result["tool_call_id"], - textwrap.indent(tool_result["output"], " "), - ( - "
\n **Error**: {}\n".format( - tool_result["exception"] - ) - if tool_result["exception"] - else "" - ), - attachments, - ) - ) - attachments = attachments_by_id.get(row["id"]) - if attachments: - click.echo("\n### Attachments\n") - for i, attachment in enumerate(attachments, 1): - if attachment["path"]: - path = attachment["path"] - click.echo( - "{}. **{}**: `{}`".format(i, attachment["type"], path) - ) - elif attachment["url"]: - click.echo( - "{}. **{}**: {}".format( - i, attachment["type"], attachment["url"] - ) - ) - elif attachment["content_length"]: - click.echo( - "{}. **{}**: `<{} bytes>`".format( - i, - attachment["type"], - f"{attachment['content_length']:,}", - ) - ) - - # If a schema was provided and the row is valid JSON, pretty print and syntax highlight it - response = row["response"] - if row["schema_json"]: - try: - parsed = json.loads(response) - response = "```json\n{}\n```".format(json.dumps(parsed, indent=2)) - except ValueError: - pass - click.echo("\n## Response\n") - if row["tool_calls"]: - click.echo("### Tool calls\n") - for tool_call in row["tool_calls"]: - click.echo( - "- **{}**: `{}`
\n Arguments: {}".format( - tool_call["name"], - tool_call["tool_call_id"], - json.dumps(tool_call["arguments"]), - ) - ) - click.echo("") - if response: - click.echo("{}\n".format(response)) - if usage: - token_usage = token_usage_string( - row["input_tokens"], - row["output_tokens"], - json.loads(row["token_details"]) if row["token_details"] else None, - ) - if token_usage: - click.echo("## Token usage\n\n{}\n".format(token_usage)) - - -@cli.group( - cls=DefaultGroup, - default="list", - default_if_no_args=True, -) -def models(): - "Manage available models" - - -_type_lookup = { - "number": "float", - "integer": "int", - "string": "str", - "object": "dict", -} - - -@models.command(name="list") -@click.option( - "--options", is_flag=True, help="Show options for each model, if available" -) -@click.option("async_", "--async", is_flag=True, help="List async models") -@click.option("--schemas", is_flag=True, help="List models that support schemas") -@click.option("--tools", is_flag=True, help="List models that support tools") -@click.option( - "-q", - "--query", - multiple=True, - help="Search for models matching these strings", -) -@click.option("model_ids", "-m", "--model", help="Specific model IDs", multiple=True) -def models_list(options, async_, schemas, tools, query, model_ids): - "List available models" - models_that_have_shown_options = set() - for model_with_aliases in get_models_with_aliases(): - if async_ and not model_with_aliases.async_model: - continue - if query: - # Only show models where every provided query string matches - if not all(model_with_aliases.matches(q) for q in query): - continue - if model_ids: - ids_and_aliases = set( - [model_with_aliases.model.model_id] + model_with_aliases.aliases - ) - if not ids_and_aliases.intersection(model_ids): - continue - if schemas and not model_with_aliases.model.supports_schema: - continue - if tools and not model_with_aliases.model.supports_tools: - continue - extra_info = [] - if model_with_aliases.aliases: - extra_info.append( - "aliases: {}".format(", ".join(model_with_aliases.aliases)) - ) - model = ( - model_with_aliases.model if not async_ else model_with_aliases.async_model - ) - output = str(model) - if extra_info: - output += " ({})".format(", ".join(extra_info)) - if options and model.Options.model_json_schema()["properties"]: - output += "\n Options:" - for name, field in model.Options.model_json_schema()["properties"].items(): - any_of = field.get("anyOf") - if any_of is None: - any_of = [{"type": field.get("type", "str")}] - types = ", ".join( - [ - _type_lookup.get(item.get("type"), item.get("type", "str")) - for item in any_of - if item.get("type") != "null" - ] - ) - bits = ["\n ", name, ": ", types] - description = field.get("description", "") - if description and ( - model.__class__ not in models_that_have_shown_options - ): - wrapped = textwrap.wrap(description, 70) - bits.append("\n ") - bits.extend("\n ".join(wrapped)) - output += "".join(bits) - models_that_have_shown_options.add(model.__class__) - if options and model.attachment_types: - attachment_types = ", ".join(sorted(model.attachment_types)) - wrapper = textwrap.TextWrapper( - width=min(max(shutil.get_terminal_size().columns, 30), 70), - initial_indent=" ", - subsequent_indent=" ", - ) - output += "\n Attachment types:\n{}".format(wrapper.fill(attachment_types)) - features = ( - [] - + (["streaming"] if model.can_stream else []) - + (["schemas"] if model.supports_schema else []) - + (["tools"] if model.supports_tools else []) - + (["async"] if model_with_aliases.async_model else []) - ) - if options and features: - output += "\n Features:\n{}".format( - "\n".join(" - {}".format(feature) for feature in features) - ) - if options and hasattr(model, "needs_key") and model.needs_key: - output += "\n Keys:" - if hasattr(model, "needs_key") and model.needs_key: - output += "\n key: {}".format(model.needs_key) - if hasattr(model, "key_env_var") and model.key_env_var: - output += "\n env_var: {}".format(model.key_env_var) - click.echo(output) - if not query and not options and not schemas and not model_ids: - click.echo(f"Default: {get_default_model()}") - - -@models.command(name="default") -@click.argument("model", required=False) -def models_default(model): - "Show or set the default model" - if not model: - click.echo(get_default_model()) - return - # Validate it is a known model - try: - model = get_model(model) - set_default_model(model.model_id) - except KeyError: - raise click.ClickException("Unknown model: {}".format(model)) - - -@cli.group( - cls=DefaultGroup, - default="list", - default_if_no_args=True, -) -def templates(): - "Manage stored prompt templates" - - -@templates.command(name="list") -def templates_list(): - "List available prompt templates" - path = template_dir() - pairs = [] - for file in path.glob("*.yaml"): - name = file.stem - try: - template = load_template(name) - except LoadTemplateError: - # Skip invalid templates - continue - text = [] - if template.system: - text.append(f"system: {template.system}") - if template.prompt: - text.append(f" prompt: {template.prompt}") - else: - text = [template.prompt if template.prompt else ""] - pairs.append((name, "".join(text).replace("\n", " "))) - try: - max_name_len = max(len(p[0]) for p in pairs) - except ValueError: - return - else: - fmt = "{name:<" + str(max_name_len) + "} : {prompt}" - for name, prompt in sorted(pairs): - text = fmt.format(name=name, prompt=prompt) - click.echo(display_truncated(text)) - - -@templates.command(name="show") -@click.argument("name") -def templates_show(name): - "Show the specified prompt template" - try: - template = load_template(name) - except LoadTemplateError: - raise click.ClickException(f"Template '{name}' not found or invalid") - click.echo( - yaml.dump( - dict((k, v) for k, v in template.model_dump().items() if v is not None), - indent=4, - default_flow_style=False, - ) - ) - - -@templates.command(name="edit") -@click.argument("name") -def templates_edit(name): - "Edit the specified prompt template using the default $EDITOR" - # First ensure it exists - path = template_dir() / f"{name}.yaml" - if not path.exists(): - path.write_text(DEFAULT_TEMPLATE, "utf-8") - click.edit(filename=str(path)) - # Validate that template - load_template(name) - - -@templates.command(name="path") -def templates_path(): - "Output the path to the templates directory" - click.echo(template_dir()) - - -@templates.command(name="loaders") -def templates_loaders(): - "Show template loaders registered by plugins" - found = False - for prefix, loader in get_template_loaders().items(): - found = True - docs = "Undocumented" - if loader.__doc__: - docs = textwrap.dedent(loader.__doc__).strip() - click.echo(f"{prefix}:") - click.echo(textwrap.indent(docs, " ")) - if not found: - click.echo("No template loaders found") - - -@cli.group( - cls=DefaultGroup, - default="list", - default_if_no_args=True, -) -def schemas(): - "Manage stored schemas" - - -@schemas.command(name="list") -@click.option( - "-p", - "--path", - type=click.Path(readable=True, exists=True, dir_okay=False), - help="Path to log database", - hidden=True, -) -@click.option( - "-d", - "--database", - type=click.Path(readable=True, exists=True, dir_okay=False), - help="Path to log database", -) -@click.option( - "queries", - "-q", - "--query", - multiple=True, - help="Search for schemas matching this string", -) -@click.option("--full", is_flag=True, help="Output full schema contents") -@click.option("json_", "--json", is_flag=True, help="Output as JSON") -@click.option("nl", "--nl", is_flag=True, help="Output as newline-delimited JSON") -def schemas_list(path, database, queries, full, json_, nl): - "List stored schemas" - if database and not path: - path = database - path = pathlib.Path(path or logs_db_path()) - if not path.exists(): - raise click.ClickException("No log database found at {}".format(path)) - db = sqlite_utils.Database(path) - migrate(db) - - params = [] - where_sql = "" - if queries: - where_bits = ["schemas.content like ?" for _ in queries] - where_sql += " where {}".format(" and ".join(where_bits)) - params.extend("%{}%".format(q) for q in queries) - - sql = """ - select - schemas.id, - schemas.content, - max(responses.datetime_utc) as recently_used, - count(*) as times_used - from schemas - join responses - on responses.schema_id = schemas.id - {} group by responses.schema_id - order by recently_used - """.format(where_sql) - rows = db.query(sql, params) - - if json_ or nl: - for line in output_rows_as_json(rows, json_cols={"content"}, nl=nl): - click.echo(line) - return - - for row in rows: - click.echo("- id: {}".format(row["id"])) - if full: - click.echo( - " schema: |\n{}".format( - textwrap.indent( - json.dumps(json.loads(row["content"]), indent=2), " " - ) - ) - ) - else: - click.echo( - " summary: |\n {}".format( - schema_summary(json.loads(row["content"])) - ) - ) - click.echo( - " usage: |\n {} time{}, most recently {}".format( - row["times_used"], - "s" if row["times_used"] != 1 else "", - row["recently_used"], - ) - ) - - -@schemas.command(name="show") -@click.argument("schema_id") -@click.option( - "-p", - "--path", - type=click.Path(readable=True, exists=True, dir_okay=False), - help="Path to log database", - hidden=True, -) -@click.option( - "-d", - "--database", - type=click.Path(readable=True, exists=True, dir_okay=False), - help="Path to log database", -) -def schemas_show(schema_id, path, database): - "Show a stored schema" - if database and not path: - path = database - path = pathlib.Path(path or logs_db_path()) - if not path.exists(): - raise click.ClickException("No log database found at {}".format(path)) - db = sqlite_utils.Database(path) - migrate(db) - - try: - row = db["schemas"].get(schema_id) - except sqlite_utils.db.NotFoundError: - raise click.ClickException("Invalid schema ID") - click.echo(json.dumps(json.loads(row["content"]), indent=2)) - - -@schemas.command(name="dsl") -@click.argument("input") -@click.option("--multi", is_flag=True, help="Wrap in an array") -def schemas_dsl_debug(input, multi): - """ - Convert LLM's schema DSL to a JSON schema - - \b - llm schema dsl 'name, age int, bio: their bio' - """ - schema = schema_dsl(input, multi) - click.echo(json.dumps(schema, indent=2)) - - -@cli.group( - cls=DefaultGroup, - default="list", - default_if_no_args=True, -) -def tools(): - "Manage tools that can be made available to LLMs" - - -@tools.command(name="list") -@click.argument("tool_defs", nargs=-1) -@click.option("json_", "--json", is_flag=True, help="Output as JSON") -@click.option( - "python_tools", - "--functions", - help="Python code block or file path defining functions to register as tools", - multiple=True, -) -def tools_list(tool_defs, json_, python_tools): - "List available tools that have been provided by plugins" - - def introspect_tools(toolbox_class): - methods = [] - for tool in toolbox_class.method_tools(): - methods.append( - { - "name": tool.name, - "description": tool.description, - "arguments": tool.input_schema, - "implementation": tool.implementation, - } - ) - return methods - - if tool_defs: - tools = {} - for tool in _gather_tools(tool_defs, python_tools): - if hasattr(tool, "name"): - tools[tool.name] = tool - else: - tools[tool.__class__.__name__] = tool - else: - tools = get_tools() - if python_tools: - for code_or_path in python_tools: - for tool in _tools_from_code(code_or_path): - tools[tool.name] = tool - - output_tools = [] - output_toolboxes = [] - tool_objects = [] - toolbox_objects = [] - for name, tool in sorted(tools.items()): - if isinstance(tool, Tool): - tool_objects.append(tool) - output_tools.append( - { - "name": name, - "description": tool.description, - "arguments": tool.input_schema, - "plugin": tool.plugin, - } - ) - else: - toolbox_objects.append(tool) - output_toolboxes.append( - { - "name": name, - "tools": [ - { - "name": tool["name"], - "description": tool["description"], - "arguments": tool["arguments"], - } - for tool in introspect_tools(tool) - ], - } - ) - if json_: - click.echo( - json.dumps( - {"tools": output_tools, "toolboxes": output_toolboxes}, - indent=2, - ) - ) - else: - for tool in tool_objects: - sig = "()" - if tool.implementation: - sig = str(inspect.signature(tool.implementation)) - click.echo( - "{}{}{}\n".format( - tool.name, - sig, - " (plugin: {})".format(tool.plugin) if tool.plugin else "", - ) - ) - if tool.description: - click.echo(textwrap.indent(tool.description.strip(), " ") + "\n") - for toolbox in toolbox_objects: - click.echo(toolbox.name + ":\n") - for tool in toolbox.method_tools(): - sig = ( - str(inspect.signature(tool.implementation)) - .replace("(self, ", "(") - .replace("(self)", "()") - ) - click.echo( - " {}{}\n".format( - tool.name, - sig, - ) - ) - if tool.description: - click.echo(textwrap.indent(tool.description.strip(), " ") + "\n") - - -@cli.group( - cls=DefaultGroup, - default="list", - default_if_no_args=True, -) -def aliases(): - "Manage model aliases" - - -@aliases.command(name="list") -@click.option("json_", "--json", is_flag=True, help="Output as JSON") -def aliases_list(json_): - "List current aliases" - to_output = [] - for alias, model in get_model_aliases().items(): - if alias != model.model_id: - to_output.append((alias, model.model_id, "")) - for alias, embedding_model in get_embedding_model_aliases().items(): - if alias != embedding_model.model_id: - to_output.append((alias, embedding_model.model_id, "embedding")) - if json_: - click.echo( - json.dumps({key: value for key, value, type_ in to_output}, indent=4) - ) - return - max_alias_length = max(len(a) for a, _, _ in to_output) - fmt = "{alias:<" + str(max_alias_length) + "} : {model_id}{type_}" - for alias, model_id, type_ in to_output: - click.echo( - fmt.format( - alias=alias, model_id=model_id, type_=f" ({type_})" if type_ else "" - ) - ) - - -@aliases.command(name="set") -@click.argument("alias") -@click.argument("model_id", required=False) -@click.option( - "-q", - "--query", - multiple=True, - help="Set alias for model matching these strings", -) -def aliases_set(alias, model_id, query): - """ - Set an alias for a model - - Example usage: - - \b - llm aliases set mini gpt-4o-mini - - Alternatively you can omit the model ID and specify one or more -q options. - The first model matching all of those query strings will be used. - - \b - llm aliases set mini -q 4o -q mini - """ - if not model_id: - if not query: - raise click.ClickException( - "You must provide a model_id or at least one -q option" - ) - # Search for the first model matching all query strings - found = None - for model_with_aliases in get_models_with_aliases(): - if all(model_with_aliases.matches(q) for q in query): - found = model_with_aliases - break - if not found: - raise click.ClickException( - "No model found matching query: " + ", ".join(query) - ) - model_id = found.model.model_id - set_alias(alias, model_id) - click.echo( - f"Alias '{alias}' set to model '{model_id}'", - err=True, - ) - else: - set_alias(alias, model_id) - - -@aliases.command(name="remove") -@click.argument("alias") -def aliases_remove(alias): - """ - Remove an alias - - Example usage: - - \b - $ llm aliases remove turbo - """ - try: - remove_alias(alias) - except KeyError as ex: - raise click.ClickException(ex.args[0]) - - -@aliases.command(name="path") -def aliases_path(): - "Output the path to the aliases.json file" - click.echo(user_dir() / "aliases.json") - - -@cli.group( - cls=DefaultGroup, - default="list", - default_if_no_args=True, -) -def fragments(): - """ - Manage fragments that are stored in the database - - Fragments are reusable snippets of text that are shared across multiple prompts. - """ - - -@fragments.command(name="list") -@click.option( - "queries", - "-q", - "--query", - multiple=True, - help="Search for fragments matching these strings", -) -@click.option("--aliases", is_flag=True, help="Show only fragments with aliases") -@click.option("json_", "--json", is_flag=True, help="Output as JSON") -def fragments_list(queries, aliases, json_): - "List current fragments" - db = sqlite_utils.Database(logs_db_path()) - migrate(db) - params = {} - param_count = 0 - where_bits = [] - if aliases: - where_bits.append("fragment_aliases.alias is not null") - for q in queries: - param_count += 1 - p = f"p{param_count}" - params[p] = q - where_bits.append(f""" - (fragments.hash = :{p} or fragment_aliases.alias = :{p} - or fragments.source like '%' || :{p} || '%' - or fragments.content like '%' || :{p} || '%') - """) - where = "\n and\n ".join(where_bits) - if where: - where = " where " + where - sql = """ - select - fragments.hash, - json_group_array(fragment_aliases.alias) filter ( - where - fragment_aliases.alias is not null - ) as aliases, - fragments.datetime_utc, - fragments.source, - fragments.content - from - fragments - left join - fragment_aliases on fragment_aliases.fragment_id = fragments.id - {where} - group by - fragments.id, fragments.hash, fragments.content, fragments.datetime_utc, fragments.source - order by fragments.datetime_utc - """.format(where=where) - results = list(db.query(sql, params)) - for result in results: - result["aliases"] = json.loads(result["aliases"]) - if json_: - click.echo(json.dumps(results, indent=4)) - else: - yaml.add_representer( - str, - lambda dumper, data: dumper.represent_scalar( - "tag:yaml.org,2002:str", data, style="|" if "\n" in data else None - ), - ) - for result in results: - result["content"] = truncate_string(result["content"]) - click.echo(yaml.dump([result], sort_keys=False, width=sys.maxsize).strip()) - - -@fragments.command(name="set") -@click.argument("alias", callback=validate_fragment_alias) -@click.argument("fragment") -def fragments_set(alias, fragment): - """ - Set an alias for a fragment - - Accepts an alias and a file path, URL, hash or '-' for stdin - - Example usage: - - \b - llm fragments set mydocs ./docs.md - """ - db = sqlite_utils.Database(logs_db_path()) - migrate(db) - try: - resolved = resolve_fragments(db, [fragment])[0] - except FragmentNotFound as ex: - raise click.ClickException(str(ex)) - migrate(db) - alias_sql = """ - insert into fragment_aliases (alias, fragment_id) - values (:alias, :fragment_id) - on conflict(alias) do update set - fragment_id = excluded.fragment_id; - """ - with db.conn: - fragment_id = ensure_fragment(db, resolved) - db.conn.execute(alias_sql, {"alias": alias, "fragment_id": fragment_id}) - - -@fragments.command(name="show") -@click.argument("alias_or_hash") -def fragments_show(alias_or_hash): - """ - Display the fragment stored under an alias or hash - - \b - llm fragments show mydocs - """ - db = sqlite_utils.Database(logs_db_path()) - migrate(db) - try: - resolved = resolve_fragments(db, [alias_or_hash])[0] - except FragmentNotFound as ex: - raise click.ClickException(str(ex)) - click.echo(resolved) - - -@fragments.command(name="remove") -@click.argument("alias", callback=validate_fragment_alias) -def fragments_remove(alias): - """ - Remove a fragment alias - - Example usage: - - \b - llm fragments remove docs - """ - db = sqlite_utils.Database(logs_db_path()) - migrate(db) - with db.conn: - db.conn.execute( - "delete from fragment_aliases where alias = :alias", {"alias": alias} - ) - - -@fragments.command(name="loaders") -def fragments_loaders(): - """Show fragment loaders registered by plugins""" - from llm import get_fragment_loaders - - found = False - for prefix, loader in get_fragment_loaders().items(): - if found: - # Extra newline on all after the first - click.echo("") - found = True - docs = "Undocumented" - if loader.__doc__: - docs = textwrap.dedent(loader.__doc__).strip() - click.echo(f"{prefix}:") - click.echo(textwrap.indent(docs, " ")) - if not found: - click.echo("No fragment loaders found") - - -@cli.command(name="plugins") -@click.option("--all", help="Include built-in default plugins", is_flag=True) -@click.option( - "hooks", "--hook", help="Filter for plugins that implement this hook", multiple=True -) -def plugins_list(all, hooks): - "List installed plugins" - plugins = get_plugins(all) - hooks = set(hooks) - if hooks: - plugins = [plugin for plugin in plugins if hooks.intersection(plugin["hooks"])] - click.echo(json.dumps(plugins, indent=2)) - - -def display_truncated(text): - console_width = shutil.get_terminal_size()[0] - if len(text) > console_width: - return text[: console_width - 3] + "..." - else: - return text - - -@cli.command() -@click.argument("packages", nargs=-1, required=False) -@click.option( - "-U", "--upgrade", is_flag=True, help="Upgrade packages to latest version" -) -@click.option( - "-e", - "--editable", - help="Install a project in editable mode from this path", -) -@click.option( - "--force-reinstall", - is_flag=True, - help="Reinstall all packages even if they are already up-to-date", -) -@click.option( - "--no-cache-dir", - is_flag=True, - help="Disable the cache", -) -@click.option( - "--pre", - is_flag=True, - help="Include pre-release and development versions", -) -def install(packages, upgrade, editable, force_reinstall, no_cache_dir, pre): - """Install packages from PyPI into the same environment as LLM""" - args = ["pip", "install"] - if upgrade: - args += ["--upgrade"] - if editable: - args += ["--editable", editable] - if force_reinstall: - args += ["--force-reinstall"] - if no_cache_dir: - args += ["--no-cache-dir"] - if pre: - args += ["--pre"] - args += list(packages) - sys.argv = args - run_module("pip", run_name="__main__") - - -@cli.command() -@click.argument("packages", nargs=-1, required=True) -@click.option("-y", "--yes", is_flag=True, help="Don't ask for confirmation") -def uninstall(packages, yes): - """Uninstall Python packages from the LLM environment""" - sys.argv = ["pip", "uninstall"] + list(packages) + (["-y"] if yes else []) - run_module("pip", run_name="__main__") - - -@cli.command() -@click.argument("collection", required=False) -@click.argument("id", required=False) -@click.option( - "-i", - "--input", - type=click.Path(exists=True, readable=True, allow_dash=True), - help="File to embed", -) -@click.option( - "-m", "--model", help="Embedding model to use", envvar="LLM_EMBEDDING_MODEL" -) -@click.option("--store", is_flag=True, help="Store the text itself in the database") -@click.option( - "-d", - "--database", - type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), - envvar="LLM_EMBEDDINGS_DB", -) -@click.option( - "-c", - "--content", - help="Content to embed", -) -@click.option("--binary", is_flag=True, help="Treat input as binary data") -@click.option( - "--metadata", - help="JSON object metadata to store", - callback=json_validator("metadata"), -) -@click.option( - "format_", - "-f", - "--format", - type=click.Choice(["json", "blob", "base64", "hex"]), - help="Output format", -) -def embed( - collection, id, input, model, store, database, content, binary, metadata, format_ -): - """Embed text and store or return the result""" - if collection and not id: - raise click.ClickException("Must provide both collection and id") - - if store and not collection: - raise click.ClickException("Must provide collection when using --store") - - # Lazy load this because we do not need it for -c or -i versions - def get_db(): - if database: - return sqlite_utils.Database(database) - else: - return sqlite_utils.Database(user_dir() / "embeddings.db") - - collection_obj = None - model_obj = None - if collection: - db = get_db() - if Collection.exists(db, collection): - # Load existing collection and use its model - collection_obj = Collection(collection, db) - model_obj = collection_obj.model() - else: - # We will create a new one, but that means model is required - if not model: - model = get_default_embedding_model() - if model is None: - raise click.ClickException( - "You need to specify an embedding model (no default model is set)" - ) - collection_obj = Collection(collection, db=db, model_id=model) - model_obj = collection_obj.model() - - if model_obj is None: - if model is None: - model = get_default_embedding_model() - try: - model_obj = get_embedding_model(model) - except UnknownModelError: - raise click.ClickException( - "You need to specify an embedding model (no default model is set)" - ) - - show_output = True - if collection and (format_ is None): - show_output = False - - # Resolve input text - if not content: - if not input or input == "-": - # Read from stdin - input_source = sys.stdin.buffer if binary else sys.stdin - content = input_source.read() - else: - mode = "rb" if binary else "r" - with open(input, mode) as f: - content = f.read() - - if not content: - raise click.ClickException("No content provided") - - if collection_obj: - embedding = collection_obj.embed(id, content, metadata=metadata, store=store) - else: - embedding = model_obj.embed(content) - - if show_output: - if format_ == "json" or format_ is None: - click.echo(json.dumps(embedding)) - elif format_ == "blob": - click.echo(encode(embedding)) - elif format_ == "base64": - click.echo(base64.b64encode(encode(embedding)).decode("ascii")) - elif format_ == "hex": - click.echo(encode(embedding).hex()) - - -@cli.command() -@click.argument("collection") -@click.argument( - "input_path", - type=click.Path(exists=True, dir_okay=False, allow_dash=True, readable=True), - required=False, -) -@click.option( - "--format", - type=click.Choice(["json", "csv", "tsv", "nl"]), - help="Format of input file - defaults to auto-detect", -) -@click.option( - "--files", - type=(click.Path(file_okay=False, dir_okay=True, allow_dash=False), str), - multiple=True, - help="Embed files in this directory - specify directory and glob pattern", -) -@click.option( - "encodings", - "--encoding", - help="Encodings to try when reading --files", - multiple=True, -) -@click.option("--binary", is_flag=True, help="Treat --files as binary data") -@click.option("--sql", help="Read input using this SQL query") -@click.option( - "--attach", - type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)), - multiple=True, - help="Additional databases to attach - specify alias and file path", -) -@click.option( - "--batch-size", type=int, help="Batch size to use when running embeddings" -) -@click.option("--prefix", help="Prefix to add to the IDs", default="") -@click.option( - "-m", "--model", help="Embedding model to use", envvar="LLM_EMBEDDING_MODEL" -) -@click.option( - "--prepend", - help="Prepend this string to all content before embedding", -) -@click.option("--store", is_flag=True, help="Store the text itself in the database") -@click.option( - "-d", - "--database", - type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), - envvar="LLM_EMBEDDINGS_DB", -) -def embed_multi( - collection, - input_path, - format, - files, - encodings, - binary, - sql, - attach, - batch_size, - prefix, - model, - prepend, - store, - database, -): - """ - Store embeddings for multiple strings at once in the specified collection. - - Input data can come from one of three sources: - - \b - 1. A CSV, TSV, JSON or JSONL file: - - CSV/TSV: First column is ID, remaining columns concatenated as content - - JSON: Array of objects with "id" field and content fields - - JSONL: Newline-delimited JSON objects - - \b - Examples: - llm embed-multi docs input.csv - cat data.json | llm embed-multi docs - - llm embed-multi docs input.json --format json - - \b - 2. A SQL query against a SQLite database: - - First column returned is used as ID - - Other columns concatenated to form content - - \b - Examples: - llm embed-multi docs --sql "SELECT id, title, body FROM posts" - llm embed-multi docs --attach blog blog.db --sql "SELECT id, content FROM blog.posts" - - \b - 3. Files in directories matching glob patterns: - - Each file becomes one embedding - - Relative file paths become IDs - - \b - Examples: - llm embed-multi docs --files docs '**/*.md' - llm embed-multi images --files photos '*.jpg' --binary - llm embed-multi texts --files texts '*.txt' --encoding utf-8 --encoding latin-1 - """ - if binary and not files: - raise click.UsageError("--binary must be used with --files") - if binary and encodings: - raise click.UsageError("--binary cannot be used with --encoding") - if not input_path and not sql and not files: - raise click.UsageError("Either --sql or input path or --files is required") - - if files: - if input_path or sql or format: - raise click.UsageError( - "Cannot use --files with --sql, input path or --format" - ) - - if database: - db = sqlite_utils.Database(database) - else: - db = sqlite_utils.Database(user_dir() / "embeddings.db") - - for alias, attach_path in attach: - db.attach(alias, attach_path) - - try: - collection_obj = Collection( - collection, db=db, model_id=model or get_default_embedding_model() - ) - except ValueError: - raise click.ClickException( - "You need to specify an embedding model (no default model is set)" - ) - - expected_length = None - if files: - encodings = encodings or ("utf-8", "latin-1") - - def count_files(): - i = 0 - for directory, pattern in files: - for path in pathlib.Path(directory).glob(pattern): - i += 1 - return i - - def iterate_files(): - for directory, pattern in files: - p = pathlib.Path(directory) - if not p.exists() or not p.is_dir(): - # fixes issue/274 - raise error if directory does not exist - raise click.UsageError(f"Invalid directory: {directory}") - for path in pathlib.Path(directory).glob(pattern): - if path.is_dir(): - continue # fixed issue/280 - skip directories - relative = path.relative_to(directory) - content = None - if binary: - content = path.read_bytes() - else: - for encoding in encodings: - try: - content = path.read_text(encoding=encoding) - except UnicodeDecodeError: - continue - if content is None: - # Log to stderr - click.echo( - "Could not decode text in file {}".format(path), - err=True, - ) - else: - yield {"id": str(relative), "content": content} - - expected_length = count_files() - rows = iterate_files() - elif sql: - rows = db.query(sql) - count_sql = "select count(*) as c from ({})".format(sql) - expected_length = next(db.query(count_sql))["c"] - else: - - def load_rows(fp): - return rows_from_file(fp, Format[format.upper()] if format else None)[0] - - try: - if input_path != "-": - # Read the file twice - first time is to get a count - expected_length = 0 - with open(input_path, "rb") as fp: - for _ in load_rows(fp): - expected_length += 1 - - rows = load_rows( - open(input_path, "rb") - if input_path != "-" - else io.BufferedReader(sys.stdin.buffer) - ) - except json.JSONDecodeError as ex: - raise click.ClickException(str(ex)) - - with click.progressbar( - rows, label="Embedding", show_percent=True, length=expected_length - ) as rows: - - def tuples() -> Iterable[Tuple[str, Union[bytes, str]]]: - for row in rows: - values = list(row.values()) - id: str = prefix + str(values[0]) - content: Optional[Union[bytes, str]] = None - if binary: - content = cast(bytes, values[1]) - else: - content = " ".join(v or "" for v in values[1:]) - if prepend and isinstance(content, str): - content = prepend + content - yield id, content or "" - - embed_kwargs = {"store": store} - if batch_size: - embed_kwargs["batch_size"] = batch_size - collection_obj.embed_multi(tuples(), **embed_kwargs) - - -@cli.command() -@click.argument("collection") -@click.argument("id", required=False) -@click.option( - "-i", - "--input", - type=click.Path(exists=True, readable=True, allow_dash=True), - help="File to embed for comparison", -) -@click.option("-c", "--content", help="Content to embed for comparison") -@click.option("--binary", is_flag=True, help="Treat input as binary data") -@click.option( - "-n", "--number", type=int, default=10, help="Number of results to return" -) -@click.option("-p", "--plain", is_flag=True, help="Output in plain text format") -@click.option( - "-d", - "--database", - type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), - envvar="LLM_EMBEDDINGS_DB", -) -@click.option("--prefix", help="Just IDs with this prefix", default="") -def similar(collection, id, input, content, binary, number, plain, database, prefix): - """ - Return top N similar IDs from a collection using cosine similarity. - - Example usage: - - \b - llm similar my-collection -c "I like cats" - - Or to find content similar to a specific stored ID: - - \b - llm similar my-collection 1234 - """ - if not id and not content and not input: - raise click.ClickException("Must provide content or an ID for the comparison") - - if database: - db = sqlite_utils.Database(database) - else: - db = sqlite_utils.Database(user_dir() / "embeddings.db") - - if not db["embeddings"].exists(): - raise click.ClickException("No embeddings table found in database") - - try: - collection_obj = Collection(collection, db, create=False) - except Collection.DoesNotExist: - raise click.ClickException("Collection does not exist") - - if id: - try: - results = collection_obj.similar_by_id(id, number, prefix=prefix) - except Collection.DoesNotExist: - raise click.ClickException("ID not found in collection") - else: - # Resolve input text - if not content: - if not input or input == "-": - # Read from stdin - input_source = sys.stdin.buffer if binary else sys.stdin - content = input_source.read() - else: - mode = "rb" if binary else "r" - with open(input, mode) as f: - content = f.read() - if not content: - raise click.ClickException("No content provided") - results = collection_obj.similar(content, number, prefix=prefix) - - for result in results: - if plain: - click.echo(f"{result.id} ({result.score})\n") - if result.content: - click.echo(textwrap.indent(result.content, " ")) - if result.metadata: - click.echo(textwrap.indent(json.dumps(result.metadata), " ")) - click.echo("") - else: - click.echo(json.dumps(asdict(result))) - - -@cli.group( - cls=DefaultGroup, - default="list", - default_if_no_args=True, -) -def embed_models(): - "Manage available embedding models" - - -@embed_models.command(name="list") -@click.option( - "-q", - "--query", - multiple=True, - help="Search for embedding models matching these strings", -) -def embed_models_list(query): - "List available embedding models" - output = [] - for model_with_aliases in get_embedding_models_with_aliases(): - if query: - if not all(model_with_aliases.matches(q) for q in query): - continue - s = str(model_with_aliases.model) - if model_with_aliases.aliases: - s += " (aliases: {})".format(", ".join(model_with_aliases.aliases)) - output.append(s) - click.echo("\n".join(output)) - - -@embed_models.command(name="default") -@click.argument("model", required=False) -@click.option( - "--remove-default", is_flag=True, help="Reset to specifying no default model" -) -def embed_models_default(model, remove_default): - "Show or set the default embedding model" - if not model and not remove_default: - default = get_default_embedding_model() - if default is None: - click.echo("", err=True) - else: - click.echo(default) - return - # Validate it is a known model - try: - if remove_default: - set_default_embedding_model(None) - else: - model = get_embedding_model(model) - set_default_embedding_model(model.model_id) - except KeyError: - raise click.ClickException("Unknown embedding model: {}".format(model)) - - -@cli.group( - cls=DefaultGroup, - default="list", - default_if_no_args=True, -) -def collections(): - "View and manage collections of embeddings" - - -@collections.command(name="path") -def collections_path(): - "Output the path to the embeddings database" - click.echo(user_dir() / "embeddings.db") - - -@collections.command(name="list") -@click.option( - "-d", - "--database", - type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), - envvar="LLM_EMBEDDINGS_DB", - help="Path to embeddings database", -) -@click.option("json_", "--json", is_flag=True, help="Output as JSON") -def embed_db_collections(database, json_): - "View a list of collections" - database = database or (user_dir() / "embeddings.db") - db = sqlite_utils.Database(str(database)) - if not db["collections"].exists(): - raise click.ClickException("No collections table found in {}".format(database)) - rows = db.query(""" - select - collections.name, - collections.model, - count(embeddings.id) as num_embeddings - from - collections left join embeddings - on collections.id = embeddings.collection_id - group by - collections.name, collections.model - """) - if json_: - click.echo(json.dumps(list(rows), indent=4)) - else: - for row in rows: - click.echo("{}: {}".format(row["name"], row["model"])) - click.echo( - " {} embedding{}".format( - row["num_embeddings"], "s" if row["num_embeddings"] != 1 else "" - ) - ) - - -@collections.command(name="delete") -@click.argument("collection") -@click.option( - "-d", - "--database", - type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), - envvar="LLM_EMBEDDINGS_DB", - help="Path to embeddings database", -) -def collections_delete(collection, database): - """ - Delete the specified collection - - Example usage: - - \b - llm collections delete my-collection - """ - database = database or (user_dir() / "embeddings.db") - db = sqlite_utils.Database(str(database)) - try: - collection_obj = Collection(collection, db, create=False) - except Collection.DoesNotExist: - raise click.ClickException("Collection does not exist") - collection_obj.delete() - - -@models.group( - cls=DefaultGroup, - default="list", - default_if_no_args=True, -) -def options(): - "Manage default options for models" - - -@options.command(name="list") -def options_list(): - """ - List default options for all models - - Example usage: - - \b - llm models options list - """ - options = get_all_model_options() - if not options: - click.echo("No default options set for any models.", err=True) - return - - for model_id, model_options in options.items(): - click.echo(f"{model_id}:") - for key, value in model_options.items(): - click.echo(f" {key}: {value}") - - -@options.command(name="show") -@click.argument("model") -def options_show(model): - """ - List default options set for a specific model - - Example usage: - - \b - llm models options show gpt-4o - """ - import llm - - try: - # Resolve alias to model ID - model_obj = llm.get_model(model) - model_id = model_obj.model_id - except llm.UnknownModelError: - # Use as-is if not found - model_id = model - - options = get_model_options(model_id) - if not options: - click.echo(f"No default options set for model '{model_id}'.", err=True) - return - - for key, value in options.items(): - click.echo(f"{key}: {value}") - - -@options.command(name="set") -@click.argument("model") -@click.argument("key") -@click.argument("value") -def options_set(model, key, value): - """ - Set a default option for a model - - Example usage: - - \b - llm models options set gpt-4o temperature 0.5 - """ - import llm - - try: - # Resolve alias to model ID - model_obj = llm.get_model(model) - model_id = model_obj.model_id - - # Validate option against model schema - try: - # Create a test Options object to validate - test_options = {key: value} - model_obj.Options(**test_options) - except pydantic.ValidationError as ex: - raise click.ClickException(render_errors(ex.errors())) - - except llm.UnknownModelError: - # Use as-is if not found - model_id = model - - set_model_option(model_id, key, value) - click.echo(f"Set default option {key}={value} for model {model_id}", err=True) - - -@options.command(name="clear") -@click.argument("model") -@click.argument("key", required=False) -def options_clear(model, key): - """ - Clear default option(s) for a model - - Example usage: - - \b - llm models options clear gpt-4o - # Or for a single option - llm models options clear gpt-4o temperature - """ - import llm - - try: - # Resolve alias to model ID - model_obj = llm.get_model(model) - model_id = model_obj.model_id - except llm.UnknownModelError: - # Use as-is if not found - model_id = model - - cleared_keys = [] - if not key: - cleared_keys = list(get_model_options(model_id).keys()) - for key_ in cleared_keys: - clear_model_option(model_id, key_) - else: - cleared_keys.append(key) - clear_model_option(model_id, key) - if cleared_keys: - if len(cleared_keys) == 1: - click.echo(f"Cleared option '{cleared_keys[0]}' for model {model_id}") - else: - click.echo( - f"Cleared {', '.join(cleared_keys)} options for model {model_id}" - ) - - -def template_dir(): - path = user_dir() / "templates" - path.mkdir(parents=True, exist_ok=True) - return path - - -def logs_db_path(): - return user_dir() / "logs.db" - - -def get_history(chat_id): - if chat_id is None: - return None, [] - log_path = logs_db_path() - db = sqlite_utils.Database(log_path) - migrate(db) - if chat_id == -1: - # Return the most recent chat - last_row = list(db["logs"].rows_where(order_by="-id", limit=1)) - if last_row: - chat_id = last_row[0].get("chat_id") or last_row[0].get("id") - else: # Database is empty - return None, [] - rows = db["logs"].rows_where( - "id = ? or chat_id = ?", [chat_id, chat_id], order_by="id" - ) - return chat_id, rows - - -def render_errors(errors): - output = [] - for error in errors: - output.append(", ".join(error["loc"])) - output.append(" " + error["msg"]) - return "\n".join(output) - - -load_plugins() - -pm.hook.register_commands(cli=cli) - - -def _human_readable_size(size_bytes): - if size_bytes == 0: - return "0B" - - size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") - i = 0 - - while size_bytes >= 1024 and i < len(size_name) - 1: - size_bytes /= 1024.0 - i += 1 - - return "{:.2f}{}".format(size_bytes, size_name[i]) - - -def logs_on(): - return not (user_dir() / "logs-off").exists() - - -def get_all_model_options() -> dict: - """ - Get all default options for all models - """ - path = user_dir() / "model_options.json" - if not path.exists(): - return {} - - try: - options = json.loads(path.read_text()) - except json.JSONDecodeError: - return {} - - return options - - -def get_model_options(model_id: str) -> dict: - """ - Get default options for a specific model - - Args: - model_id: Return options for model with this ID - - Returns: - A dictionary of model options - """ - path = user_dir() / "model_options.json" - if not path.exists(): - return {} - - try: - options = json.loads(path.read_text()) - except json.JSONDecodeError: - return {} - - return options.get(model_id, {}) - - -def set_model_option(model_id: str, key: str, value: Any) -> None: - """ - Set a default option for a model. - - Args: - model_id: The model ID - key: The option key - value: The option value - """ - path = user_dir() / "model_options.json" - if path.exists(): - try: - options = json.loads(path.read_text()) - except json.JSONDecodeError: - options = {} - else: - options = {} - - # Ensure the model has an entry - if model_id not in options: - options[model_id] = {} - - # Set the option - options[model_id][key] = value - - # Save the options - path.write_text(json.dumps(options, indent=2)) - - -def clear_model_option(model_id: str, key: str) -> None: - """ - Clear a model option - - Args: - model_id: The model ID - key: Key to clear - """ - path = user_dir() / "model_options.json" - if not path.exists(): - return - - try: - options = json.loads(path.read_text()) - except json.JSONDecodeError: - return - - if model_id not in options: - return - - if key in options[model_id]: - del options[model_id][key] - if not options[model_id]: - del options[model_id] - - path.write_text(json.dumps(options, indent=2)) - - -class LoadTemplateError(ValueError): - pass - - -def _parse_yaml_template(name, content): - try: - loaded = yaml.safe_load(content) - except yaml.YAMLError as ex: - raise LoadTemplateError("Invalid YAML: {}".format(str(ex))) - if isinstance(loaded, str): - return Template(name=name, prompt=loaded) - loaded["name"] = name - try: - return Template(**loaded) - except pydantic.ValidationError as ex: - msg = "A validation error occurred:\n" - msg += render_errors(ex.errors()) - raise LoadTemplateError(msg) - - -def load_template(name: str) -> Template: - "Load template, or raise LoadTemplateError(msg)" - if name.startswith("https://") or name.startswith("http://"): - response = httpx.get(name) - try: - response.raise_for_status() - except httpx.HTTPStatusError as ex: - raise LoadTemplateError("Could not load template {}: {}".format(name, ex)) - return _parse_yaml_template(name, response.text) - - potential_path = pathlib.Path(name) - - if has_plugin_prefix(name) and not potential_path.exists(): - prefix, rest = name.split(":", 1) - loaders = get_template_loaders() - if prefix not in loaders: - raise LoadTemplateError("Unknown template prefix: {}".format(prefix)) - loader = loaders[prefix] - try: - return loader(rest) - except Exception as ex: - raise LoadTemplateError("Could not load template {}: {}".format(name, ex)) - - # Try local file - if potential_path.exists(): - path = potential_path - else: - # Look for template in template_dir() - path = template_dir() / f"{name}.yaml" - if not path.exists(): - raise LoadTemplateError(f"Invalid template: {name}") - content = path.read_text() - template_obj = _parse_yaml_template(name, content) - # We trust functions here because they came from the filesystem - template_obj._functions_is_trusted = True - return template_obj - - -def _tools_from_code(code_or_path: str) -> List[Tool]: - """ - Treat all Python functions in the code as tools - """ - if "\n" not in code_or_path and code_or_path.endswith(".py"): - try: - code_or_path = pathlib.Path(code_or_path).read_text() - except FileNotFoundError: - raise click.ClickException("File not found: {}".format(code_or_path)) - namespace: Dict[str, Any] = {} - tools = [] - try: - exec(code_or_path, namespace) - except SyntaxError as ex: - raise click.ClickException("Error in --functions definition: {}".format(ex)) - # Register all callables in the locals dict: - for name, value in namespace.items(): - if callable(value) and not name.startswith("_"): - tools.append(Tool.function(value)) - return tools - - -def _debug_tool_call(_, tool_call, tool_result): - click.echo( - click.style( - "\nTool call: {}({})".format(tool_call.name, tool_call.arguments), - fg="yellow", - bold=True, - ), - err=True, - ) - output = "" - attachments = "" - if tool_result.attachments: - attachments += "\nAttachments:\n" - for attachment in tool_result.attachments: - attachments += f" {repr(attachment)}\n" - - try: - output = json.dumps(json.loads(tool_result.output), indent=2) - except ValueError: - output = tool_result.output - output += attachments - click.echo( - click.style( - textwrap.indent(output, " ") + ("\n" if not tool_result.exception else ""), - fg="green", - bold=True, - ), - err=True, - ) - if tool_result.exception: - click.echo( - click.style( - " Exception: {}".format(tool_result.exception), - fg="red", - bold=True, - ), - err=True, - ) - - -def _approve_tool_call(_, tool_call): - click.echo( - click.style( - "Tool call: {}({})".format(tool_call.name, tool_call.arguments), - fg="yellow", - bold=True, - ), - err=True, - ) - if not click.confirm("Approve tool call?"): - raise CancelToolCall("User cancelled tool call") - - -def _gather_tools( - tool_specs: List[str], python_tools: List[str] -) -> List[Union[Tool, Type[Toolbox]]]: - tools: List[Union[Tool, Type[Toolbox]]] = [] - if python_tools: - for code_or_path in python_tools: - tools.extend(_tools_from_code(code_or_path)) - registered_tools = get_tools() - registered_classes = dict( - (key, value) - for key, value in registered_tools.items() - if inspect.isclass(value) - ) - bad_tools = [ - tool for tool in tool_specs if tool.split("(")[0] not in registered_tools - ] - if bad_tools: - raise click.ClickException( - "Tool(s) {} not found. Available tools: {}".format( - ", ".join(bad_tools), ", ".join(registered_tools.keys()) - ) - ) - for tool_spec in tool_specs: - if not tool_spec[0].isupper(): - # It's a function - tools.append(registered_tools[tool_spec]) - else: - # It's a class - tools.append(instantiate_from_spec(registered_classes, tool_spec)) - return tools - - -def _get_conversation_tools(conversation, tools): - if conversation and not tools and conversation.responses: - # Copy plugin tools from first response in conversation - initial_tools = conversation.responses[0].prompt.tools - if initial_tools: - # Only tools from plugins: - return [tool.name for tool in initial_tools if tool.plugin] diff --git a/build/lib/llm/default_plugins/__init__.py b/build/lib/llm/default_plugins/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/build/lib/llm/default_plugins/default_tools.py b/build/lib/llm/default_plugins/default_tools.py deleted file mode 100644 index 53ff72cd0..000000000 --- a/build/lib/llm/default_plugins/default_tools.py +++ /dev/null @@ -1,8 +0,0 @@ -import llm -from llm.tools import llm_time, llm_version - - -@llm.hookimpl -def register_tools(register): - register(llm_version) - register(llm_time) diff --git a/build/lib/llm/default_plugins/openai_models.py b/build/lib/llm/default_plugins/openai_models.py deleted file mode 100644 index 9a7013390..000000000 --- a/build/lib/llm/default_plugins/openai_models.py +++ /dev/null @@ -1,1212 +0,0 @@ -from llm import ( - AsyncConversation, - AsyncKeyModel, - AsyncResponse, - Conversation, - EmbeddingModel, - KeyModel, - Prompt, - Response, - StreamEvent, - hookimpl, -) -import llm -from llm.utils import ( - dicts_to_table_string, - remove_dict_none_values, - logging_client, - simplify_usage_dict, -) -import click -import datetime -from enum import Enum -import httpx -import openai -import os - -from pydantic import field_validator, Field - -from typing import AsyncGenerator, cast, List, Iterable, Iterator, Optional, Union -import json -import yaml - - -@hookimpl -def register_models(register): - # GPT-4o - register( - Chat("gpt-4o", vision=True, supports_schema=True, supports_tools=True), - AsyncChat("gpt-4o", vision=True, supports_schema=True, supports_tools=True), - aliases=("4o",), - ) - register( - Chat("chatgpt-4o-latest", vision=True), - AsyncChat("chatgpt-4o-latest", vision=True), - aliases=("chatgpt-4o",), - ) - register( - Chat("gpt-4o-mini", vision=True, supports_schema=True, supports_tools=True), - AsyncChat( - "gpt-4o-mini", vision=True, supports_schema=True, supports_tools=True - ), - aliases=("4o-mini",), - ) - for audio_model_id in ( - "gpt-4o-audio-preview", - "gpt-4o-audio-preview-2024-12-17", - "gpt-4o-audio-preview-2024-10-01", - "gpt-4o-mini-audio-preview", - "gpt-4o-mini-audio-preview-2024-12-17", - ): - register( - Chat(audio_model_id, audio=True), - AsyncChat(audio_model_id, audio=True), - ) - # GPT-4.1 - for model_id in ("gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano"): - register( - Chat(model_id, vision=True, supports_schema=True, supports_tools=True), - AsyncChat(model_id, vision=True, supports_schema=True, supports_tools=True), - aliases=(model_id.replace("gpt-", ""),), - ) - # 3.5 and 4 - register( - Chat("gpt-3.5-turbo"), AsyncChat("gpt-3.5-turbo"), aliases=("3.5", "chatgpt") - ) - register( - Chat("gpt-3.5-turbo-16k"), - AsyncChat("gpt-3.5-turbo-16k"), - aliases=("chatgpt-16k", "3.5-16k"), - ) - register(Chat("gpt-4"), AsyncChat("gpt-4"), aliases=("4", "gpt4")) - register(Chat("gpt-4-32k"), AsyncChat("gpt-4-32k"), aliases=("4-32k",)) - # GPT-4 Turbo models - register(Chat("gpt-4-1106-preview"), AsyncChat("gpt-4-1106-preview")) - register(Chat("gpt-4-0125-preview"), AsyncChat("gpt-4-0125-preview")) - register(Chat("gpt-4-turbo-2024-04-09"), AsyncChat("gpt-4-turbo-2024-04-09")) - register( - Chat("gpt-4-turbo"), - AsyncChat("gpt-4-turbo"), - aliases=("gpt-4-turbo-preview", "4-turbo", "4t"), - ) - # GPT-4.5 - register( - Chat( - "gpt-4.5-preview-2025-02-27", - vision=True, - supports_schema=True, - supports_tools=True, - ), - AsyncChat( - "gpt-4.5-preview-2025-02-27", - vision=True, - supports_schema=True, - supports_tools=True, - ), - ) - register( - Chat("gpt-4.5-preview", vision=True, supports_schema=True, supports_tools=True), - AsyncChat( - "gpt-4.5-preview", vision=True, supports_schema=True, supports_tools=True - ), - aliases=("gpt-4.5",), - ) - # o1 - for model_id in ("o1", "o1-2024-12-17"): - register( - Chat( - model_id, - vision=True, - can_stream=False, - reasoning=True, - supports_schema=True, - supports_tools=True, - ), - AsyncChat( - model_id, - vision=True, - can_stream=False, - reasoning=True, - supports_schema=True, - supports_tools=True, - ), - ) - - register( - Chat("o1-preview", allows_system_prompt=False), - AsyncChat("o1-preview", allows_system_prompt=False), - ) - register( - Chat("o1-mini", allows_system_prompt=False), - AsyncChat("o1-mini", allows_system_prompt=False), - ) - register( - Chat("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), - AsyncChat("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), - ) - register( - Chat( - "o3", vision=True, reasoning=True, supports_schema=True, supports_tools=True - ), - AsyncChat( - "o3", vision=True, reasoning=True, supports_schema=True, supports_tools=True - ), - ) - register( - Chat( - "o4-mini", - vision=True, - reasoning=True, - supports_schema=True, - supports_tools=True, - ), - AsyncChat( - "o4-mini", - vision=True, - reasoning=True, - supports_schema=True, - supports_tools=True, - ), - ) - # GPT-5 - for model_id in ( - "gpt-5", - "gpt-5-mini", - "gpt-5-nano", - "gpt-5-2025-08-07", - "gpt-5-mini-2025-08-07", - "gpt-5-nano-2025-08-07", - ): - register( - Chat( - model_id, - vision=True, - reasoning=True, - supports_schema=True, - supports_tools=True, - ), - AsyncChat( - model_id, - vision=True, - reasoning=True, - supports_schema=True, - supports_tools=True, - ), - ) - # GPT-5.1 - for model_id in ( - "gpt-5.1", - "gpt-5.1-chat-latest", - ): - register( - Chat( - model_id, - vision=True, - reasoning=True, - supports_schema=True, - supports_tools=True, - ), - AsyncChat( - model_id, - vision=True, - reasoning=True, - supports_schema=True, - supports_tools=True, - ), - ) - # GPT-5.2 - for model_id in ("gpt-5.2", "gpt-5.2-chat-latest"): - register( - Chat( - model_id, - vision=True, - reasoning=True, - supports_schema=True, - supports_tools=True, - ), - AsyncChat( - model_id, - vision=True, - reasoning=True, - supports_schema=True, - supports_tools=True, - ), - ) - # "gpt-5.2-pro" is Responses API only - - # GPT-5.4 - for model_id in ( - "gpt-5.4", - "gpt-5.4-2026-03-05", - "gpt-5.4-mini", - "gpt-5.4-mini-2026-03-17", - "gpt-5.4-nano", - "gpt-5.4-nano-2026-03-17", - ): - register( - Chat( - model_id, - vision=True, - reasoning=True, - supports_schema=True, - supports_tools=True, - ), - AsyncChat( - model_id, - vision=True, - reasoning=True, - supports_schema=True, - supports_tools=True, - ), - ) - - # The -instruct completion model - register( - Completion("gpt-3.5-turbo-instruct", default_max_tokens=256), - aliases=("3.5-instruct", "chatgpt-instruct"), - ) - - # Load extra models - extra_path = llm.user_dir() / "extra-openai-models.yaml" - if not extra_path.exists(): - return - with open(extra_path) as f: - extra_models = yaml.safe_load(f) - for extra_model in extra_models: - model_id = extra_model["model_id"] - aliases = extra_model.get("aliases", []) - model_name = extra_model["model_name"] - api_base = extra_model.get("api_base") - api_type = extra_model.get("api_type") - api_version = extra_model.get("api_version") - api_engine = extra_model.get("api_engine") - headers = extra_model.get("headers") - reasoning = extra_model.get("reasoning") - kwargs = {} - if extra_model.get("can_stream") is False: - kwargs["can_stream"] = False - if extra_model.get("supports_schema") is True: - kwargs["supports_schema"] = True - if extra_model.get("supports_tools") is True: - kwargs["supports_tools"] = True - if extra_model.get("vision") is True: - kwargs["vision"] = True - if extra_model.get("audio") is True: - kwargs["audio"] = True - if extra_model.get("completion"): - klass = Completion - async_klass = None - else: - klass = Chat - async_klass = AsyncChat - model_kwargs = dict( - model_id=model_id, - model_name=model_name, - api_base=api_base, - api_type=api_type, - api_version=api_version, - api_engine=api_engine, - headers=headers, - reasoning=reasoning, - **kwargs, - ) - chat_model = klass(**model_kwargs) - async_model = async_klass(**model_kwargs) if async_klass else None - if api_base: - chat_model.needs_key = None - if async_model: - async_model.needs_key = None - if extra_model.get("api_key_name"): - chat_model.needs_key = extra_model["api_key_name"] - if async_model: - async_model.needs_key = extra_model["api_key_name"] - register( - chat_model, - async_model, - aliases=aliases, - ) - - -@hookimpl -def register_embedding_models(register): - register( - OpenAIEmbeddingModel("text-embedding-ada-002", "text-embedding-ada-002"), - aliases=( - "ada", - "ada-002", - ), - ) - register( - OpenAIEmbeddingModel("text-embedding-3-small", "text-embedding-3-small"), - aliases=("3-small",), - ) - register( - OpenAIEmbeddingModel("text-embedding-3-large", "text-embedding-3-large"), - aliases=("3-large",), - ) - # With varying dimensions - register( - OpenAIEmbeddingModel( - "text-embedding-3-small-512", "text-embedding-3-small", 512 - ), - aliases=("3-small-512",), - ) - register( - OpenAIEmbeddingModel( - "text-embedding-3-large-256", "text-embedding-3-large", 256 - ), - aliases=("3-large-256",), - ) - register( - OpenAIEmbeddingModel( - "text-embedding-3-large-1024", "text-embedding-3-large", 1024 - ), - aliases=("3-large-1024",), - ) - - -class OpenAIEmbeddingModel(EmbeddingModel): - needs_key = "openai" - key_env_var = "OPENAI_API_KEY" - batch_size = 100 - - def __init__(self, model_id, openai_model_id, dimensions=None): - self.model_id = model_id - self.openai_model_id = openai_model_id - self.dimensions = dimensions - - def embed_batch(self, items: Iterable[Union[str, bytes]]) -> Iterator[List[float]]: - kwargs = { - "input": items, - "model": self.openai_model_id, - } - if self.dimensions: - kwargs["dimensions"] = self.dimensions - client = openai.OpenAI(api_key=self.get_key()) - results = client.embeddings.create(**kwargs).data - return ([float(r) for r in result.embedding] for result in results) - - -@hookimpl -def register_commands(cli): - @cli.group(name="openai") - def openai_(): - "Commands for working directly with the OpenAI API" - - @openai_.command() - @click.option("json_", "--json", is_flag=True, help="Output as JSON") - @click.option("--key", help="OpenAI API key") - def models(json_, key): - "List models available to you from the OpenAI API" - from llm import get_key - - api_key = get_key(key, "openai", "OPENAI_API_KEY") - response = httpx.get( - "https://api.openai.com/v1/models", - headers={"Authorization": f"Bearer {api_key}"}, - ) - if response.status_code != 200: - raise click.ClickException( - f"Error {response.status_code} from OpenAI API: {response.text}" - ) - models = response.json()["data"] - if json_: - click.echo(json.dumps(models, indent=4)) - else: - to_print = [] - for model in models: - # Print id, owned_by, root, created as ISO 8601 - created_str = datetime.datetime.fromtimestamp( - model["created"], datetime.timezone.utc - ).isoformat() - to_print.append( - { - "id": model["id"], - "owned_by": model["owned_by"], - "created": created_str, - } - ) - done = dicts_to_table_string("id owned_by created".split(), to_print) - print("\n".join(done)) - - -class SharedOptions(llm.Options): - temperature: Optional[float] = Field( - description=( - "What sampling temperature to use, between 0 and 2. Higher values like " - "0.8 will make the output more random, while lower values like 0.2 will " - "make it more focused and deterministic." - ), - ge=0, - le=2, - default=None, - ) - max_tokens: Optional[int] = Field( - description="Maximum number of tokens to generate.", default=None - ) - top_p: Optional[float] = Field( - description=( - "An alternative to sampling with temperature, called nucleus sampling, " - "where the model considers the results of the tokens with top_p " - "probability mass. So 0.1 means only the tokens comprising the top " - "10% probability mass are considered. Recommended to use top_p or " - "temperature but not both." - ), - ge=0, - le=1, - default=None, - ) - frequency_penalty: Optional[float] = Field( - description=( - "Number between -2.0 and 2.0. Positive values penalize new tokens based " - "on their existing frequency in the text so far, decreasing the model's " - "likelihood to repeat the same line verbatim." - ), - ge=-2, - le=2, - default=None, - ) - presence_penalty: Optional[float] = Field( - description=( - "Number between -2.0 and 2.0. Positive values penalize new tokens based " - "on whether they appear in the text so far, increasing the model's " - "likelihood to talk about new topics." - ), - ge=-2, - le=2, - default=None, - ) - stop: Optional[str] = Field( - description=("A string where the API will stop generating further tokens."), - default=None, - ) - logit_bias: Optional[Union[dict, str]] = Field( - description=( - "Modify the likelihood of specified tokens appearing in the completion. " - 'Pass a JSON string like \'{"1712":-100, "892":-100, "1489":-100}\'' - ), - default=None, - ) - seed: Optional[int] = Field( - description="Integer seed to attempt to sample deterministically", - default=None, - ) - - @field_validator("logit_bias") - def validate_logit_bias(cls, logit_bias): - if logit_bias is None: - return None - - if isinstance(logit_bias, str): - try: - logit_bias = json.loads(logit_bias) - except json.JSONDecodeError: - raise ValueError("Invalid JSON in logit_bias string") - - validated_logit_bias = {} - for key, value in logit_bias.items(): - try: - int_key = int(key) - int_value = int(value) - if -100 <= int_value <= 100: - validated_logit_bias[int_key] = int_value - else: - raise ValueError("Value must be between -100 and 100") - except ValueError: - raise ValueError("Invalid key-value pair in logit_bias dictionary") - - return validated_logit_bias - - -class ReasoningEffortEnum(str, Enum): - none = "none" - minimal = "minimal" - low = "low" - medium = "medium" - high = "high" - xhigh = "xhigh" - - -class OptionsForReasoning(SharedOptions): - json_object: Optional[bool] = Field( - description="Output a valid JSON object {...}. Prompt must mention JSON.", - default=None, - ) - reasoning_effort: Optional[ReasoningEffortEnum] = Field( - description=( - "Constraints effort on reasoning for reasoning models. Currently supported " - "values are low, medium, and high. Reducing reasoning effort can result in " - "faster responses and fewer tokens used on reasoning in a response." - ), - default=None, - ) - - -def _attachment(attachment): - url = attachment.url - base64_content = "" - if not url or attachment.resolve_type().startswith("audio/"): - base64_content = attachment.base64_content() - url = f"data:{attachment.resolve_type()};base64,{base64_content}" - if attachment.resolve_type() == "application/pdf": - if not base64_content: - base64_content = attachment.base64_content() - return { - "type": "file", - "file": { - "filename": f"{attachment.id()}.pdf", - "file_data": f"data:application/pdf;base64,{base64_content}", - }, - } - if attachment.resolve_type().startswith("image/"): - return {"type": "image_url", "image_url": {"url": url}} - else: - format_ = "wav" if attachment.resolve_type() == "audio/wav" else "mp3" - return { - "type": "input_audio", - "input_audio": { - "data": base64_content, - "format": format_, - }, - } - - -class _Shared: - def __init__( - self, - model_id, - key=None, - model_name=None, - api_base=None, - api_type=None, - api_version=None, - api_engine=None, - headers=None, - can_stream=True, - vision=False, - audio=False, - reasoning=False, - supports_schema=False, - supports_tools=False, - allows_system_prompt=True, - ): - self.model_id = model_id - self.key = key - self.supports_schema = supports_schema - self.supports_tools = supports_tools - self.model_name = model_name - self.api_base = api_base - self.api_type = api_type - self.api_version = api_version - self.api_engine = api_engine - self.headers = headers - self.can_stream = can_stream - self.vision = vision - self.allows_system_prompt = allows_system_prompt - - self.attachment_types = set() - - if reasoning: - self.Options = OptionsForReasoning - - if vision: - self.attachment_types.update( - { - "image/png", - "image/jpeg", - "image/webp", - "image/gif", - "application/pdf", - } - ) - - if audio: - self.attachment_types.update( - { - "audio/wav", - "audio/mpeg", - } - ) - - def __str__(self) -> str: - return "OpenAI Chat: {}".format(self.model_id) - - def _append_llm_message(self, out, message, current_system): - """Translate one llm.Message into one (or more) OpenAI message - dicts and append them to ``out``. - - Returns the (possibly updated) current_system value so the caller - can dedup consecutive identical system messages. - """ - from llm.parts import ( - AttachmentPart, - TextPart, - ToolCallPart, - ToolResultPart, - ) - - text_bits = [] - attachment_items = [] - tool_calls = [] - tool_results = [] - - for part in message.parts: - if isinstance(part, TextPart): - text_bits.append(part.text) - elif isinstance(part, AttachmentPart) and part.attachment: - attachment_items.append(_attachment(part.attachment)) - elif isinstance(part, ToolCallPart): - tool_calls.append( - { - "type": "function", - "id": part.tool_call_id, - "function": { - "name": part.name, - "arguments": json.dumps(part.arguments), - }, - } - ) - elif isinstance(part, ToolResultPart): - tool_results.append( - { - "role": "tool", - "tool_call_id": part.tool_call_id, - "content": part.output, - } - ) - - # Role "tool" emits one OpenAI "tool" message per ToolResultPart. - if message.role == "tool": - out.extend(tool_results) - return current_system - - # System dedup — skip if we just emitted this exact system text. - if message.role == "system": - text = "".join(text_bits) - if text == current_system: - return current_system - current_system = text - - if attachment_items: - content = [] - if text_bits: - content.append({"type": "text", "text": "".join(text_bits)}) - content.extend(attachment_items) - entry = {"role": message.role, "content": content} - else: - entry = { - "role": message.role, - "content": "".join(text_bits) if text_bits else None, - } - - if tool_calls: - entry["tool_calls"] = tool_calls - # OpenAI expects content=null when only tool_calls are present. - if not text_bits: - entry["content"] = None - elif entry["content"] is None and message.role != "assistant": - # For user/system, an empty message is pointless — drop it. - return current_system - - out.append(entry) - return current_system - - def build_messages(self, prompt, conversation): - """Translate prompt.messages into OpenAI's wire format. - - Under the Phase 7 invariant, ``prompt.messages`` is the full - chain for this turn — Conversation.prompt and response.reply - pre-bake the history into it. The ``conversation`` parameter - is unused and retained only for the plugin API contract. - """ - messages: List[Dict[str, Any]] = [] - current_system: Optional[str] = None - for msg in prompt.messages: - current_system = self._append_llm_message( - messages, msg, current_system - ) - return messages - - def set_usage(self, response, usage): - if not usage: - return - input_tokens = usage.pop("prompt_tokens") - output_tokens = usage.pop("completion_tokens") - usage.pop("total_tokens") - response.set_usage( - input=input_tokens, output=output_tokens, details=simplify_usage_dict(usage) - ) - - def get_client(self, key, *, async_=False): - kwargs = {} - if self.api_base: - kwargs["base_url"] = self.api_base - if self.api_type: - kwargs["api_type"] = self.api_type - if self.api_version: - kwargs["api_version"] = self.api_version - if self.api_engine: - kwargs["engine"] = self.api_engine - if self.needs_key: - kwargs["api_key"] = self.get_key(key) - else: - # OpenAI-compatible models don't need a key, but the - # openai client library requires one - kwargs["api_key"] = "DUMMY_KEY" - if self.headers: - kwargs["default_headers"] = self.headers - if os.environ.get("LLM_OPENAI_SHOW_RESPONSES"): - kwargs["http_client"] = logging_client() - if async_: - return openai.AsyncOpenAI(**kwargs) - else: - return openai.OpenAI(**kwargs) - - def build_kwargs(self, prompt, stream): - kwargs = dict(not_nulls(prompt.options)) - json_object = kwargs.pop("json_object", None) - if "max_tokens" not in kwargs and self.default_max_tokens is not None: - kwargs["max_tokens"] = self.default_max_tokens - if json_object: - kwargs["response_format"] = {"type": "json_object"} - if prompt.schema: - kwargs["response_format"] = { - "type": "json_schema", - "json_schema": {"name": "output", "schema": prompt.schema}, - } - if prompt.tools: - kwargs["tools"] = [ - { - "type": "function", - "function": { - "name": tool.name, - "description": tool.description or None, - "parameters": tool.input_schema, - }, - } - for tool in prompt.tools - ] - if stream: - kwargs["stream_options"] = {"include_usage": True} - return kwargs - - -class Chat(_Shared, KeyModel): - needs_key = "openai" - key_env_var = "OPENAI_API_KEY" - default_max_tokens = None - - class Options(SharedOptions): - json_object: Optional[bool] = Field( - description="Output a valid JSON object {...}. Prompt must mention JSON.", - default=None, - ) - - def execute( - self, - prompt: Prompt, - stream: bool, - response: Response, - conversation: Optional[Conversation] = None, - key: Optional[str] = None, - ) -> Iterator[str]: - if prompt.system and not self.allows_system_prompt: - raise NotImplementedError("Model does not support system prompts") - messages = self.build_messages(prompt, conversation) - kwargs = self.build_kwargs(prompt, stream) - client = self.get_client(key) - usage = None - if stream: - completion = client.chat.completions.create( - model=self.model_name or self.model_id, - messages=messages, - stream=True, - **kwargs, - ) - chunks = [] - tool_calls = {} - # part_index allocator. Text always uses 0. Each tool call - # at delta index i is assigned a part_index past any text - # that was seen, so _build_parts groups them correctly. - seen_text = False - tc_part_index = {} - next_part_index = 1 - for chunk in completion: - chunks.append(chunk) - if chunk.usage: - usage = chunk.usage.model_dump() - if chunk.choices and chunk.choices[0].delta: - for tool_call in chunk.choices[0].delta.tool_calls or []: - if tool_call.function.arguments is None: - tool_call.function.arguments = "" - idx = tool_call.index - if idx not in tool_calls: - tool_calls[idx] = tool_call - tc_part_index[idx] = next_part_index - next_part_index += 1 - yield StreamEvent( - type="tool_call_name", - chunk=tool_call.function.name or "", - part_index=tc_part_index[idx], - tool_call_id=tool_call.id, - ) - else: - tool_calls[idx].function.arguments += ( - tool_call.function.arguments - ) - if tool_call.function.arguments: - yield StreamEvent( - type="tool_call_args", - chunk=tool_call.function.arguments, - part_index=tc_part_index[idx], - tool_call_id=tool_calls[idx].id, - ) - try: - content = chunk.choices[0].delta.content - except IndexError: - content = None - if content: - # Empty strings are noise (OpenAI's first chunk - # with role=assistant has content=""). - seen_text = True - yield StreamEvent( - type="text", chunk=content, part_index=0 - ) - response.response_json = remove_dict_none_values(combine_chunks(chunks)) - if tool_calls: - for value in tool_calls.values(): - response.add_tool_call( - llm.ToolCall( - tool_call_id=value.id, - name=value.function.name, - arguments=json.loads(value.function.arguments), - ) - ) - else: - completion = client.chat.completions.create( - model=self.model_name or self.model_id, - messages=messages, - stream=False, - **kwargs, - ) - usage = completion.usage.model_dump() - response.response_json = remove_dict_none_values(completion.model_dump()) - part_index = 0 - for tool_call in completion.choices[0].message.tool_calls or []: - response.add_tool_call( - llm.ToolCall( - tool_call_id=tool_call.id, - name=tool_call.function.name, - arguments=json.loads(tool_call.function.arguments), - ) - ) - part_index += 1 - yield StreamEvent( - type="tool_call_name", - chunk=tool_call.function.name or "", - part_index=part_index, - tool_call_id=tool_call.id, - ) - yield StreamEvent( - type="tool_call_args", - chunk=tool_call.function.arguments or "", - part_index=part_index, - tool_call_id=tool_call.id, - ) - if completion.choices[0].message.content is not None: - yield StreamEvent( - type="text", - chunk=completion.choices[0].message.content, - part_index=0, - ) - # Capture the reasoning token count BEFORE set_usage runs — - # set_usage pops top-level keys and passes the rest through - # simplify_usage_dict, which strips zero-valued entries. - if usage: - reasoning_tokens = ( - (usage.get("completion_tokens_details") or {}).get( - "reasoning_tokens", 0 - ) - ) - if reasoning_tokens: - response._reasoning_token_count = reasoning_tokens - self.set_usage(response, usage) - response._prompt_json = redact_data({"messages": messages}) - - -class AsyncChat(_Shared, AsyncKeyModel): - needs_key = "openai" - key_env_var = "OPENAI_API_KEY" - default_max_tokens = None - - class Options(SharedOptions): - json_object: Optional[bool] = Field( - description="Output a valid JSON object {...}. Prompt must mention JSON.", - default=None, - ) - - async def execute( - self, - prompt: Prompt, - stream: bool, - response: AsyncResponse, - conversation: Optional[AsyncConversation] = None, - key: Optional[str] = None, - ) -> AsyncGenerator[str, None]: - if prompt.system and not self.allows_system_prompt: - raise NotImplementedError("Model does not support system prompts") - messages = self.build_messages(prompt, conversation) - kwargs = self.build_kwargs(prompt, stream) - client = self.get_client(key, async_=True) - usage = None - if stream: - completion = await client.chat.completions.create( - model=self.model_name or self.model_id, - messages=messages, - stream=True, - **kwargs, - ) - chunks = [] - tool_calls = {} - tc_part_index = {} - next_part_index = 1 - async for chunk in completion: - if chunk.usage: - usage = chunk.usage.model_dump() - chunks.append(chunk) - if chunk.choices and chunk.choices[0].delta: - for tool_call in chunk.choices[0].delta.tool_calls or []: - if tool_call.function.arguments is None: - tool_call.function.arguments = "" - idx = tool_call.index - if idx not in tool_calls: - tool_calls[idx] = tool_call - tc_part_index[idx] = next_part_index - next_part_index += 1 - yield StreamEvent( - type="tool_call_name", - chunk=tool_call.function.name or "", - part_index=tc_part_index[idx], - tool_call_id=tool_call.id, - ) - else: - tool_calls[idx].function.arguments += ( - tool_call.function.arguments - ) - if tool_call.function.arguments: - yield StreamEvent( - type="tool_call_args", - chunk=tool_call.function.arguments, - part_index=tc_part_index[idx], - tool_call_id=tool_calls[idx].id, - ) - try: - content = chunk.choices[0].delta.content - except IndexError: - content = None - if content: - yield StreamEvent( - type="text", chunk=content, part_index=0 - ) - if tool_calls: - for value in tool_calls.values(): - response.add_tool_call( - llm.ToolCall( - tool_call_id=value.id, - name=value.function.name, - arguments=json.loads(value.function.arguments), - ) - ) - response.response_json = remove_dict_none_values(combine_chunks(chunks)) - else: - completion = await client.chat.completions.create( - model=self.model_name or self.model_id, - messages=messages, - stream=False, - **kwargs, - ) - response.response_json = remove_dict_none_values(completion.model_dump()) - usage = completion.usage.model_dump() - part_index = 0 - for tool_call in completion.choices[0].message.tool_calls or []: - response.add_tool_call( - llm.ToolCall( - tool_call_id=tool_call.id, - name=tool_call.function.name, - arguments=json.loads(tool_call.function.arguments), - ) - ) - part_index += 1 - yield StreamEvent( - type="tool_call_name", - chunk=tool_call.function.name or "", - part_index=part_index, - tool_call_id=tool_call.id, - ) - yield StreamEvent( - type="tool_call_args", - chunk=tool_call.function.arguments or "", - part_index=part_index, - tool_call_id=tool_call.id, - ) - if completion.choices[0].message.content is not None: - yield StreamEvent( - type="text", - chunk=completion.choices[0].message.content, - part_index=0, - ) - # See sync Chat.execute: capture reasoning before set_usage mutates. - if usage: - reasoning_tokens = ( - (usage.get("completion_tokens_details") or {}).get( - "reasoning_tokens", 0 - ) - ) - if reasoning_tokens: - response._reasoning_token_count = reasoning_tokens - self.set_usage(response, usage) - response._prompt_json = redact_data({"messages": messages}) - - -class Completion(Chat): - class Options(SharedOptions): - logprobs: Optional[int] = Field( - description="Include the log probabilities of most likely N per token", - default=None, - le=5, - ) - - def __init__(self, *args, default_max_tokens=None, **kwargs): - super().__init__(*args, **kwargs) - self.default_max_tokens = default_max_tokens - - def __str__(self) -> str: - return "OpenAI Completion: {}".format(self.model_id) - - def execute( - self, - prompt: Prompt, - stream: bool, - response: Response, - conversation: Optional[Conversation] = None, - key: Optional[str] = None, - ) -> Iterator[str]: - if prompt.system: - raise NotImplementedError( - "System prompts are not supported for OpenAI completion models" - ) - messages = [] - if conversation is not None: - for prev_response in conversation.responses: - messages.append(prev_response.prompt.prompt) - messages.append(cast(Response, prev_response).text()) - messages.append(prompt.prompt) - kwargs = self.build_kwargs(prompt, stream) - client = self.get_client(key) - if stream: - completion = client.completions.create( - model=self.model_name or self.model_id, - prompt="\n".join(messages), - stream=True, - **kwargs, - ) - chunks = [] - for chunk in completion: - chunks.append(chunk) - try: - content = chunk.choices[0].text - except IndexError: - content = None - if content is not None: - yield content - combined = combine_chunks(chunks) - cleaned = remove_dict_none_values(combined) - response.response_json = cleaned - else: - completion = client.completions.create( - model=self.model_name or self.model_id, - prompt="\n".join(messages), - stream=False, - **kwargs, - ) - response.response_json = remove_dict_none_values(completion.model_dump()) - yield completion.choices[0].text - response._prompt_json = redact_data({"messages": messages}) - - -def not_nulls(data) -> dict: - return {key: value for key, value in data if value is not None} - - -def combine_chunks(chunks: List) -> dict: - content = "" - role = None - finish_reason = None - # If any of them have log probability, we're going to persist - # those later on - logprobs = [] - usage = {} - - for item in chunks: - if item.usage: - usage = item.usage.model_dump() - for choice in item.choices: - if choice.logprobs and hasattr(choice.logprobs, "top_logprobs"): - logprobs.append( - { - "text": choice.text if hasattr(choice, "text") else None, - "top_logprobs": choice.logprobs.top_logprobs, - } - ) - - if not hasattr(choice, "delta"): - content += choice.text - continue - role = choice.delta.role - if choice.delta.content is not None: - content += choice.delta.content - if choice.finish_reason is not None: - finish_reason = choice.finish_reason - - # Imitations of the OpenAI API may be missing some of these fields - combined = { - "content": content, - "role": role, - "finish_reason": finish_reason, - "usage": usage, - } - if logprobs: - combined["logprobs"] = logprobs - if chunks: - for key in ("id", "object", "model", "created", "index"): - value = getattr(chunks[0], key, None) - if value is not None: - combined[key] = value - - return combined - - -def redact_data(input_dict): - """ - Recursively search through the input dictionary for any 'image_url' keys - and modify the 'url' value to be just 'data:...'. - - Also redact input_audio.data keys - """ - if isinstance(input_dict, dict): - for key, value in input_dict.items(): - if ( - key == "image_url" - and isinstance(value, dict) - and "url" in value - and value["url"].startswith("data:") - ): - value["url"] = "data:..." - elif key == "input_audio" and isinstance(value, dict) and "data" in value: - value["data"] = "..." - else: - redact_data(value) - elif isinstance(input_dict, list): - for item in input_dict: - redact_data(item) - return input_dict diff --git a/build/lib/llm/embeddings.py b/build/lib/llm/embeddings.py deleted file mode 100644 index 90b983a11..000000000 --- a/build/lib/llm/embeddings.py +++ /dev/null @@ -1,367 +0,0 @@ -from .models import EmbeddingModel -from .embeddings_migrations import embeddings_migrations -from dataclasses import dataclass -import hashlib -from itertools import islice -import json -from sqlite_utils import Database -from sqlite_utils.db import Table -import time -from typing import cast, Any, Dict, Iterable, List, Optional, Tuple, Union - - -@dataclass -class Entry: - id: str - score: Optional[float] - content: Optional[str] = None - metadata: Optional[Dict[str, Any]] = None - - -class Collection: - class DoesNotExist(Exception): - pass - - def __init__( - self, - name: str, - db: Optional[Database] = None, - *, - model: Optional[EmbeddingModel] = None, - model_id: Optional[str] = None, - create: bool = True, - ) -> None: - """ - A collection of embeddings - - Returns the collection with the given name, creating it if it does not exist. - - If you set create=False a Collection.DoesNotExist exception will be raised if the - collection does not already exist. - - Args: - db (sqlite_utils.Database): Database to store the collection in - name (str): Name of the collection - model (llm.models.EmbeddingModel, optional): Embedding model to use - model_id (str, optional): Alternatively, ID of the embedding model to use - create (bool, optional): Whether to create the collection if it does not exist - """ - import llm - - self.db = db or Database(memory=True) - self.name = name - self._model = model - - embeddings_migrations.apply(self.db) - - rows = list(self.db["collections"].rows_where("name = ?", [self.name])) - if rows: - row = rows[0] - self.id = row["id"] - self.model_id = row["model"] - else: - if create: - # Collection does not exist, so model or model_id is required - if not model and not model_id: - raise ValueError( - "Either model= or model_id= must be provided when creating a new collection" - ) - # Create it - if model_id: - # Resolve alias - model = llm.get_embedding_model(model_id) - self._model = model - model_id = cast(EmbeddingModel, model).model_id - self.id = ( - cast(Table, self.db["collections"]) - .insert( - { - "name": self.name, - "model": model_id, - } - ) - .last_pk - ) - else: - raise self.DoesNotExist(f"Collection '{name}' does not exist") - - def model(self) -> EmbeddingModel: - "Return the embedding model used by this collection" - import llm - - if self._model is None: - self._model = llm.get_embedding_model(self.model_id) - - return cast(EmbeddingModel, self._model) - - def count(self) -> int: - """ - Count the number of items in the collection. - - Returns: - int: Number of items in the collection - """ - return next( - self.db.query( - """ - select count(*) as c from embeddings where collection_id = ( - select id from collections where name = ? - ) - """, - (self.name,), - ) - )["c"] - - def embed( - self, - id: str, - value: Union[str, bytes], - metadata: Optional[Dict[str, Any]] = None, - store: bool = False, - ) -> None: - """ - Embed value and store it in the collection with a given ID. - - Args: - id (str): ID for the value - value (str or bytes): value to be embedded - metadata (dict, optional): Metadata to be stored - store (bool, optional): Whether to store the value in the content or content_blob column - """ - from llm import encode - - content_hash = self.content_hash(value) - if self.db["embeddings"].count_where( - "content_hash = ? and collection_id = ?", [content_hash, self.id] - ): - return - embedding = self.model().embed(value) - cast(Table, self.db["embeddings"]).insert( - { - "collection_id": self.id, - "id": id, - "embedding": encode(embedding), - "content": value if (store and isinstance(value, str)) else None, - "content_blob": value if (store and isinstance(value, bytes)) else None, - "content_hash": content_hash, - "metadata": json.dumps(metadata) if metadata else None, - "updated": int(time.time()), - }, - replace=True, - ) - - def embed_multi( - self, - entries: Iterable[Tuple[str, Union[str, bytes]]], - store: bool = False, - batch_size: int = 100, - ) -> None: - """ - Embed multiple texts and store them in the collection with given IDs. - - Args: - entries (iterable): Iterable of (id: str, text: str) tuples - store (bool, optional): Whether to store the text in the content column - batch_size (int, optional): custom maximum batch size to use - """ - self.embed_multi_with_metadata( - ((id, value, None) for id, value in entries), - store=store, - batch_size=batch_size, - ) - - def embed_multi_with_metadata( - self, - entries: Iterable[Tuple[str, Union[str, bytes], Optional[Dict[str, Any]]]], - store: bool = False, - batch_size: int = 100, - ) -> None: - """ - Embed multiple values along with metadata and store them in the collection with given IDs. - - Args: - entries (iterable): Iterable of (id: str, value: str or bytes, metadata: None or dict) - store (bool, optional): Whether to store the value in the content or content_blob column - batch_size (int, optional): custom maximum batch size to use - """ - import llm - - batch_size = min(batch_size, (self.model().batch_size or batch_size)) - iterator = iter(entries) - collection_id = self.id - while True: - batch = list(islice(iterator, batch_size)) - if not batch: - break - # Calculate hashes first - items_and_hashes = [(item, self.content_hash(item[1])) for item in batch] - # Any of those hashes already exist? - existing_ids = [ - row["id"] - for row in self.db.query( - """ - select id from embeddings - where collection_id = ? and content_hash in ({}) - """.format(",".join("?" for _ in items_and_hashes)), - [collection_id] - + [item_and_hash[1] for item_and_hash in items_and_hashes], - ) - ] - filtered_batch = [item for item in batch if item[0] not in existing_ids] - embeddings = list( - self.model().embed_multi(item[1] for item in filtered_batch) - ) - with self.db.conn: - cast(Table, self.db["embeddings"]).insert_all( - ( - { - "collection_id": collection_id, - "id": id, - "embedding": llm.encode(embedding), - "content": ( - value if (store and isinstance(value, str)) else None - ), - "content_blob": ( - value if (store and isinstance(value, bytes)) else None - ), - "content_hash": self.content_hash(value), - "metadata": json.dumps(metadata) if metadata else None, - "updated": int(time.time()), - } - for (embedding, (id, value, metadata)) in zip( - embeddings, filtered_batch - ) - ), - replace=True, - ) - - def similar_by_vector( - self, - vector: List[float], - number: int = 10, - skip_id: Optional[str] = None, - prefix: Optional[str] = None, - ) -> List[Entry]: - """ - Find similar items in the collection by a given vector. - - Args: - vector (list): Vector to search by - number (int, optional): Number of similar items to return - skip_id (str, optional): An ID to exclude from the results - prefix: (str, optional): Filter results to IDs witih this prefix - - Returns: - list: List of Entry objects - """ - import llm - - def distance_score(other_encoded): - other_vector = llm.decode(other_encoded) - return llm.cosine_similarity(other_vector, vector) - - self.db.register_function(distance_score, replace=True) - - where_bits = ["collection_id = ?"] - where_args = [str(self.id)] - - if prefix: - where_bits.append("id LIKE ? || '%'") - where_args.append(prefix) - - if skip_id: - where_bits.append("id != ?") - where_args.append(skip_id) - - return [ - Entry( - id=row["id"], - score=row["score"], - content=row["content"], - metadata=json.loads(row["metadata"]) if row["metadata"] else None, - ) - for row in self.db.query( - """ - select id, content, metadata, distance_score(embedding) as score - from embeddings - where {where} - order by score desc limit {number} - """.format( - where=" and ".join(where_bits), - number=number, - ), - where_args, - ) - ] - - def similar_by_id( - self, id: str, number: int = 10, prefix: Optional[str] = None - ) -> List[Entry]: - """ - Find similar items in the collection by a given ID. - - Args: - id (str): ID to search by - number (int, optional): Number of similar items to return - prefix: (str, optional): Filter results to IDs with this prefix - - Returns: - list: List of Entry objects - """ - import llm - - matches = list( - self.db["embeddings"].rows_where( - "collection_id = ? and id = ?", (self.id, id) - ) - ) - if not matches: - raise self.DoesNotExist("ID not found") - embedding = matches[0]["embedding"] - comparison_vector = llm.decode(embedding) - return self.similar_by_vector( - comparison_vector, number, skip_id=id, prefix=prefix - ) - - def similar( - self, value: Union[str, bytes], number: int = 10, prefix: Optional[str] = None - ) -> List[Entry]: - """ - Find similar items in the collection by a given value. - - Args: - value (str or bytes): value to search by - number (int, optional): Number of similar items to return - prefix: (str, optional): Filter results to IDs with this prefix - - Returns: - list: List of Entry objects - """ - comparison_vector = self.model().embed(value) - return self.similar_by_vector(comparison_vector, number, prefix=prefix) - - @classmethod - def exists(cls, db: Database, name: str) -> bool: - """ - Does this collection exist in the database? - - Args: - name (str): Name of the collection - """ - rows = list(db["collections"].rows_where("name = ?", [name])) - return bool(rows) - - def delete(self): - """ - Delete the collection and its embeddings from the database - """ - with self.db.conn: - self.db.execute("delete from embeddings where collection_id = ?", [self.id]) - self.db.execute("delete from collections where id = ?", [self.id]) - - @staticmethod - def content_hash(input: Union[str, bytes]) -> bytes: - "Hash content for deduplication. Override to change hashing behavior." - if isinstance(input, str): - input = input.encode("utf8") - return hashlib.md5(input).digest() diff --git a/build/lib/llm/embeddings_migrations.py b/build/lib/llm/embeddings_migrations.py deleted file mode 100644 index 69545f3ea..000000000 --- a/build/lib/llm/embeddings_migrations.py +++ /dev/null @@ -1,89 +0,0 @@ -from sqlite_migrate import Migrations -import hashlib -import time - -embeddings_migrations = Migrations("llm.embeddings") - - -@embeddings_migrations() -def m001_create_tables(db): - db["collections"].create({"id": int, "name": str, "model": str}, pk="id") - db["collections"].create_index(["name"], unique=True) - db["embeddings"].create( - { - "collection_id": int, - "id": str, - "embedding": bytes, - "content": str, - "metadata": str, - }, - pk=("collection_id", "id"), - ) - - -@embeddings_migrations() -def m002_foreign_key(db): - db["embeddings"].add_foreign_key("collection_id", "collections", "id") - - -@embeddings_migrations() -def m003_add_updated(db): - db["embeddings"].add_column("updated", int) - # Pretty-print the schema - db["embeddings"].transform() - # Assume anything existing was last updated right now - db.query( - "update embeddings set updated = ? where updated is null", [int(time.time())] - ) - - -@embeddings_migrations() -def m004_store_content_hash(db): - db["embeddings"].add_column("content_hash", bytes) - db["embeddings"].transform( - column_order=( - "collection_id", - "id", - "embedding", - "content", - "content_hash", - "metadata", - "updated", - ) - ) - - # Register functions manually so we can de-register later - def md5(text): - return hashlib.md5(text.encode("utf8")).digest() - - def random_md5(): - return hashlib.md5(str(time.time()).encode("utf8")).digest() - - db.conn.create_function("temp_md5", 1, md5) - db.conn.create_function("temp_random_md5", 0, random_md5) - - with db.conn: - db.execute(""" - update embeddings - set content_hash = temp_md5(content) - where content is not null - """) - db.execute(""" - update embeddings - set content_hash = temp_random_md5() - where content is null - """) - - db["embeddings"].create_index(["content_hash"]) - - # De-register functions - db.conn.create_function("temp_md5", 1, None) - db.conn.create_function("temp_random_md5", 0, None) - - -@embeddings_migrations() -def m005_add_content_blob(db): - db["embeddings"].add_column("content_blob", bytes) - db["embeddings"].transform( - column_order=("collection_id", "id", "embedding", "content", "content_blob") - ) diff --git a/build/lib/llm/errors.py b/build/lib/llm/errors.py deleted file mode 100644 index 10f50bb5a..000000000 --- a/build/lib/llm/errors.py +++ /dev/null @@ -1,6 +0,0 @@ -class ModelError(Exception): - "Models can raise this error, which will be displayed to the user" - - -class NeedsKeyException(ModelError): - "Model needs an API key which has not been provided" diff --git a/build/lib/llm/hookspecs.py b/build/lib/llm/hookspecs.py deleted file mode 100644 index 7ab555199..000000000 --- a/build/lib/llm/hookspecs.py +++ /dev/null @@ -1,35 +0,0 @@ -from pluggy import HookimplMarker -from pluggy import HookspecMarker - -hookspec = HookspecMarker("llm") -hookimpl = HookimplMarker("llm") - - -@hookspec -def register_commands(cli): - """Register additional CLI commands, e.g. 'llm mycommand ...'""" - - -@hookspec -def register_models(register, model_aliases): - "Register additional model instances representing LLM models that can be called" - - -@hookspec -def register_embedding_models(register): - "Register additional model instances that can be used for embedding" - - -@hookspec -def register_template_loaders(register): - "Register additional template loaders with prefixes" - - -@hookspec -def register_fragment_loaders(register): - "Register additional fragment loaders with prefixes" - - -@hookspec -def register_tools(register): - "Register functions that can be used as tools by the LLMs" diff --git a/build/lib/llm/migrations.py b/build/lib/llm/migrations.py deleted file mode 100644 index f2ca04651..000000000 --- a/build/lib/llm/migrations.py +++ /dev/null @@ -1,420 +0,0 @@ -import datetime -from typing import Callable, List - -MIGRATIONS: List[Callable] = [] -migration = MIGRATIONS.append - - -def migrate(db): - ensure_migrations_table(db) - already_applied = {r["name"] for r in db["_llm_migrations"].rows} - for fn in MIGRATIONS: - name = fn.__name__ - if name not in already_applied: - fn(db) - db["_llm_migrations"].insert( - { - "name": name, - "applied_at": str(datetime.datetime.now(datetime.timezone.utc)), - } - ) - already_applied.add(name) - - -def ensure_migrations_table(db): - if not db["_llm_migrations"].exists(): - db["_llm_migrations"].create( - { - "name": str, - "applied_at": str, - }, - pk="name", - ) - - -@migration -def m001_initial(db): - # Ensure the original table design exists, so other migrations can run - if db["log"].exists(): - # It needs to have the chat_id column - if "chat_id" not in db["log"].columns_dict: - db["log"].add_column("chat_id") - return - db["log"].create( - { - "provider": str, - "system": str, - "prompt": str, - "chat_id": str, - "response": str, - "model": str, - "timestamp": str, - } - ) - - -@migration -def m002_id_primary_key(db): - db["log"].transform(pk="id") - - -@migration -def m003_chat_id_foreign_key(db): - db["log"].transform(types={"chat_id": int}) - db["log"].add_foreign_key("chat_id", "log", "id") - - -@migration -def m004_column_order(db): - db["log"].transform( - column_order=( - "id", - "model", - "timestamp", - "prompt", - "system", - "response", - "chat_id", - ) - ) - - -@migration -def m004_drop_provider(db): - db["log"].transform(drop=("provider",)) - - -@migration -def m005_debug(db): - db["log"].add_column("debug", str) - db["log"].add_column("duration_ms", int) - - -@migration -def m006_new_logs_table(db): - columns = db["log"].columns_dict - for column, type in ( - ("options_json", str), - ("prompt_json", str), - ("response_json", str), - ("reply_to_id", int), - ): - # It's possible people running development code like myself - # might have accidentally created these columns already - if column not in columns: - db["log"].add_column(column, type) - - # Use .transform() to rename options and timestamp_utc, and set new order - db["log"].transform( - column_order=( - "id", - "model", - "prompt", - "system", - "prompt_json", - "options_json", - "response", - "response_json", - "reply_to_id", - "chat_id", - "duration_ms", - "timestamp_utc", - ), - rename={ - "timestamp": "timestamp_utc", - "options": "options_json", - }, - ) - - -@migration -def m007_finish_logs_table(db): - db["log"].transform( - drop={"debug"}, - rename={"timestamp_utc": "datetime_utc"}, - drop_foreign_keys=("chat_id",), - ) - with db.conn: - db.execute("alter table log rename to logs") - - -@migration -def m008_reply_to_id_foreign_key(db): - db["logs"].add_foreign_key("reply_to_id", "logs", "id") - - -@migration -def m008_fix_column_order_in_logs(db): - # reply_to_id ended up at the end after foreign key added - db["logs"].transform( - column_order=( - "id", - "model", - "prompt", - "system", - "prompt_json", - "options_json", - "response", - "response_json", - "reply_to_id", - "chat_id", - "duration_ms", - "timestamp_utc", - ), - ) - - -@migration -def m009_delete_logs_table_if_empty(db): - # We moved to a new table design, but we don't delete the table - # if someone has put data in it - if not db["logs"].count: - db["logs"].drop() - - -@migration -def m010_create_new_log_tables(db): - db["conversations"].create( - { - "id": str, - "name": str, - "model": str, - }, - pk="id", - ) - db["responses"].create( - { - "id": str, - "model": str, - "prompt": str, - "system": str, - "prompt_json": str, - "options_json": str, - "response": str, - "response_json": str, - "conversation_id": str, - "duration_ms": int, - "datetime_utc": str, - }, - pk="id", - foreign_keys=(("conversation_id", "conversations", "id"),), - ) - - -@migration -def m011_fts_for_responses(db): - db["responses"].enable_fts(["prompt", "response"], create_triggers=True) - - -@migration -def m012_attachments_tables(db): - db["attachments"].create( - { - "id": str, - "type": str, - "path": str, - "url": str, - "content": bytes, - }, - pk="id", - ) - db["prompt_attachments"].create( - { - "response_id": str, - "attachment_id": str, - "order": int, - }, - foreign_keys=( - ("response_id", "responses", "id"), - ("attachment_id", "attachments", "id"), - ), - pk=("response_id", "attachment_id"), - ) - - -@migration -def m013_usage(db): - db["responses"].add_column("input_tokens", int) - db["responses"].add_column("output_tokens", int) - db["responses"].add_column("token_details", str) - - -@migration -def m014_schemas(db): - db["schemas"].create( - { - "id": str, - "content": str, - }, - pk="id", - ) - db["responses"].add_column("schema_id", str, fk="schemas", fk_col="id") - # Clean up SQL create table indentation - db["responses"].transform() - # These changes may have dropped the FTS configuration, fix that - db["responses"].enable_fts( - ["prompt", "response"], create_triggers=True, replace=True - ) - - -@migration -def m015_fragments_tables(db): - db["fragments"].create( - { - "id": int, - "hash": str, - "content": str, - "datetime_utc": str, - "source": str, - }, - pk="id", - ) - db["fragments"].create_index(["hash"], unique=True) - db["fragment_aliases"].create( - { - "alias": str, - "fragment_id": int, - }, - foreign_keys=(("fragment_id", "fragments", "id"),), - pk="alias", - ) - db["prompt_fragments"].create( - { - "response_id": str, - "fragment_id": int, - "order": int, - }, - foreign_keys=( - ("response_id", "responses", "id"), - ("fragment_id", "fragments", "id"), - ), - pk=("response_id", "fragment_id"), - ) - db["system_fragments"].create( - { - "response_id": str, - "fragment_id": int, - "order": int, - }, - foreign_keys=( - ("response_id", "responses", "id"), - ("fragment_id", "fragments", "id"), - ), - pk=("response_id", "fragment_id"), - ) - - -@migration -def m016_fragments_table_pks(db): - # The same fragment can be attached to a response multiple times - # https://github.com/simonw/llm/issues/863#issuecomment-2781720064 - db["prompt_fragments"].transform(pk=("response_id", "fragment_id", "order")) - db["system_fragments"].transform(pk=("response_id", "fragment_id", "order")) - - -@migration -def m017_tools_tables(db): - db["tools"].create( - { - "id": int, - "hash": str, - "name": str, - "description": str, - "input_schema": str, - }, - pk="id", - ) - db["tools"].create_index(["hash"], unique=True) - # Many-to-many relationship between tools and responses - db["tool_responses"].create( - { - "tool_id": int, - "response_id": str, - }, - foreign_keys=( - ("tool_id", "tools", "id"), - ("response_id", "responses", "id"), - ), - pk=("tool_id", "response_id"), - ) - # tool_calls and tool_results are one-to-many against responses - db["tool_calls"].create( - { - "id": int, - "response_id": str, - "tool_id": int, - "name": str, - "arguments": str, - "tool_call_id": str, - }, - pk="id", - foreign_keys=( - ("response_id", "responses", "id"), - ("tool_id", "tools", "id"), - ), - ) - db["tool_results"].create( - { - "id": int, - "response_id": str, - "tool_id": int, - "name": str, - "output": str, - "tool_call_id": str, - }, - pk="id", - foreign_keys=( - ("response_id", "responses", "id"), - ("tool_id", "tools", "id"), - ), - ) - - -@migration -def m017_tools_plugin(db): - db["tools"].add_column("plugin") - - -@migration -def m018_tool_instances(db): - # Used to track instances of Toolbox classes that may be - # used multiple times by different tools - db["tool_instances"].create( - { - "id": int, - "plugin": str, - "name": str, - "arguments": str, - }, - pk="id", - ) - # We record which instance was used only on the results - db["tool_results"].add_column("instance_id", fk="tool_instances") - - -@migration -def m019_resolved_model(db): - # For models like gemini-1.5-flash-latest where we wish to record - # the resolved model name in addition to the alias - db["responses"].add_column("resolved_model", str) - - -@migration -def m020_tool_results_attachments(db): - db["tool_results_attachments"].create( - { - "tool_result_id": int, - "attachment_id": str, - "order": int, - }, - foreign_keys=( - ("tool_result_id", "tool_results", "id"), - ("attachment_id", "attachments", "id"), - ), - pk=("tool_result_id", "attachment_id"), - ) - - -@migration -def m021_tool_results_exception(db): - db["tool_results"].add_column("exception", str) diff --git a/build/lib/llm/models.py b/build/lib/llm/models.py deleted file mode 100644 index 8600eb402..000000000 --- a/build/lib/llm/models.py +++ /dev/null @@ -1,2966 +0,0 @@ -import asyncio -import base64 -from condense_json import condense_json -from dataclasses import dataclass, field -import datetime -from .errors import NeedsKeyException -import hashlib -import httpx -from itertools import islice -from pathlib import Path -import re -import time -from types import MethodType -from typing import ( - Any, - AsyncGenerator, - AsyncIterator, - Awaitable, - Callable, - Dict, - Iterable, - Iterator, - List, - Optional, - Set, - Union, - get_type_hints, -) -from .serialization import ResponseDict -from .utils import ( - ensure_fragment, - ensure_tool, - make_schema_id, - mimetype_from_path, - mimetype_from_string, - token_usage_string, - monotonic_ulid, - Fragment, -) -from abc import ABC, abstractmethod -import inspect -import json -from pydantic import BaseModel, ConfigDict, create_model - -CONVERSATION_NAME_LENGTH = 32 - - -@dataclass -class Usage: - "Token usage information from a model response." - - input: Optional[int] = None - output: Optional[int] = None - details: Optional[Dict[str, Any]] = None - - -@dataclass -class Attachment: - "An attachment (image, audio, etc) to include with a prompt." - - type: Optional[str] = None - path: Optional[str] = None - url: Optional[str] = None - content: Optional[bytes] = None - _id: Optional[str] = None - - def id(self): - # Hash of the binary content, or of '{"url": "https://..."}' for URL attachments - if self._id is None: - if self.content: - self._id = hashlib.sha256(self.content).hexdigest() - elif self.path: - self._id = hashlib.sha256(Path(self.path).read_bytes()).hexdigest() - else: - self._id = hashlib.sha256( - json.dumps({"url": self.url}).encode("utf-8") - ).hexdigest() - return self._id - - def resolve_type(self): - "Return the content type, guessing from content if not specified." - if self.type: - return self.type - # Derive it from path or url or content - if self.path: - return mimetype_from_path(self.path) - if self.url: - response = httpx.head(self.url) - response.raise_for_status() - return response.headers.get("content-type") - if self.content: - return mimetype_from_string(self.content) - raise ValueError("Attachment has no type and no content to derive it from") - - def content_bytes(self): - "Return the binary content, reading from path or URL if needed." - content = self.content - if not content: - if self.path: - content = Path(self.path).read_bytes() - elif self.url: - response = httpx.get(self.url) - response.raise_for_status() - content = response.content - return content - - def base64_content(self): - "Return the content as a base64-encoded string." - return base64.b64encode(self.content_bytes()).decode("utf-8") - - def __repr__(self): - info = [f"" - - @classmethod - def from_row(cls, row): - return cls( - _id=row["id"], - type=row["type"], - path=row["path"], - url=row["url"], - content=row["content"], - ) - - -@dataclass -class Tool: - "A tool that can be called by a model." - - name: str - description: Optional[str] = None - input_schema: Dict = field(default_factory=dict) - implementation: Optional[Callable] = None - plugin: Optional[str] = None # plugin tool came from, e.g. 'llm_tools_sqlite' - - def __post_init__(self): - # Convert Pydantic model to JSON schema if needed - self.input_schema = _ensure_dict_schema(self.input_schema) - - def hash(self): - """Hash for tool based on its name, description and input schema (preserving key order)""" - to_hash = { - "name": self.name, - "description": self.description, - "input_schema": self.input_schema, - } - if self.plugin: - to_hash["plugin"] = self.plugin - return hashlib.sha256(json.dumps(to_hash).encode("utf-8")).hexdigest() - - @classmethod - def function(cls, function, name=None, description=None): - """ - Turn a Python function into a Tool object by: - - Extracting the function name - - Using the function docstring for the Tool description - - Building a Pydantic model for inputs by inspecting the function signature - - Building a Pydantic model for the return value by using the function's return annotation - """ - if not name and function.__name__ == "": - raise ValueError( - "Cannot create a Tool from a lambda function without providing name=" - ) - - return cls( - name=name or function.__name__, - description=description or function.__doc__ or None, - input_schema=_get_arguments_input_schema(function, name), - implementation=function, - ) - - -def _get_arguments_input_schema(function, name): - signature = inspect.signature(function) - type_hints = get_type_hints(function) - fields = {} - for param_name, param in signature.parameters.items(): - if param_name == "self": - continue - # Determine the type annotation (default to string if missing) - annotated_type = type_hints.get(param_name, str) - - # Handle default value if present; if there's no default, use '...' - if param.default is inspect.Parameter.empty: - fields[param_name] = (annotated_type, ...) - else: - fields[param_name] = (annotated_type, param.default) - - return create_model(f"{name}InputSchema", **fields) - - -class Toolbox: - name: Optional[str] = None - instance_id: Optional[int] = None - _blocked = ( - "tools", - "add_tool", - "method_tools", - "__init_subclass__", - "prepare", - "prepare_async", - ) - _extra_tools: List[Tool] = [] - _config: Dict[str, Any] = {} - _prepared: bool = False - _async_prepared: bool = False - - def __init_subclass__(cls, **kwargs): - super().__init_subclass__(**kwargs) - - original_init = cls.__init__ - - def wrapped_init(self, *args, **kwargs): - # Track args/kwargs passed to constructor in self._config - # so we can serialize them to a database entry later on - sig = inspect.signature(original_init) - bound = sig.bind(self, *args, **kwargs) - bound.apply_defaults() - - self._config = { - name: value - for name, value in bound.arguments.items() - if name != "self" - and sig.parameters[name].kind - not in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD) - } - self._extra_tools = [] - - original_init(self, *args, **kwargs) - - cls.__init__ = wrapped_init - - @classmethod - def method_tools(cls) -> List[Tool]: - tools = [] - for method_name in dir(cls): - if method_name.startswith("_") or method_name in cls._blocked: - continue - method = getattr(cls, method_name) - if callable(method): - tool = Tool.function( - method, - name="{}_{}".format(cls.__name__, method_name), - ) - tools.append(tool) - return tools - - def tools(self) -> Iterable[Tool]: - "Returns an llm.Tool() for each class method, plus any extras registered with add_tool()" - # method_tools() returns unbound methods, we need bound methods here: - for name in dir(self): - if name.startswith("_") or name in self._blocked: - continue - attr = getattr(self, name) - if callable(attr): - tool = Tool.function(attr, name=f"{self.__class__.__name__}_{name}") - tool.plugin = getattr(self, "plugin", None) - yield tool - yield from self._extra_tools - - def add_tool( - self, tool_or_function: Union[Tool, Callable[..., Any]], pass_self: bool = False - ): - "Add a tool to this toolbox" - - def _upgrade(fn): - if pass_self: - return MethodType(fn, self) - return fn - - if isinstance(tool_or_function, Tool): - self._extra_tools.append(tool_or_function) - elif callable(tool_or_function): - self._extra_tools.append(Tool.function(_upgrade(tool_or_function))) - else: - raise ValueError("Tool must be an instance of Tool or a callable function") - - def prepare(self): - """ - Over-ride this to perform setup (and .add_tool() calls) before the toolbox is used. - Implement a similar prepare_async() method for async setup. - """ - pass - - async def prepare_async(self): - """ - Over-ride this to perform async setup (and .add_tool() calls) before the toolbox is used. - """ - pass - - -@dataclass -class ToolCall: - "A request by the model to call a tool." - - name: str - arguments: dict - tool_call_id: Optional[str] = None - - -@dataclass -class ToolResult: - "The result of executing a tool call." - - name: str - output: str - attachments: List[Attachment] = field(default_factory=list) - tool_call_id: Optional[str] = None - instance: Optional[Toolbox] = None - exception: Optional[Exception] = None - - -@dataclass -class ToolOutput: - "Tool functions can return output with extra attachments" - - output: Optional[Union[str, dict, list, bool, int, float]] = None - attachments: List[Attachment] = field(default_factory=list) - - -ToolDef = Union[Tool, Toolbox, Callable[..., Any]] -BeforeCallSync = Callable[[Optional[Tool], ToolCall], None] -AfterCallSync = Callable[[Tool, ToolCall, ToolResult], None] -BeforeCallAsync = Callable[[Optional[Tool], ToolCall], Union[None, Awaitable[None]]] -AfterCallAsync = Callable[[Tool, ToolCall, ToolResult], Union[None, Awaitable[None]]] - - -class CancelToolCall(Exception): - pass - - -@dataclass -class Prompt: - "The prompt being sent to the model." - - _prompt: Optional[str] - model: "Model" - fragments: Optional[List[Union[str, Fragment]]] - attachments: Optional[List[Attachment]] - _system: Optional[str] - system_fragments: Optional[List[Union[str, Fragment]]] - prompt_json: Optional[str] - schema: Optional[Union[Dict, type[BaseModel]]] - tools: List[Tool] - tool_results: List[ToolResult] - options: "Options" - - def __init__( - self, - prompt, - model, - *, - fragments=None, - attachments=None, - system=None, - system_fragments=None, - prompt_json=None, - options=None, - schema=None, - tools=None, - tool_results=None, - messages=None, - ): - self._prompt = prompt - self.model = model - self.attachments = list(attachments or []) - self.fragments = fragments or [] - self._system = system - self.system_fragments = system_fragments or [] - self.prompt_json = prompt_json - if schema and not isinstance(schema, dict) and issubclass(schema, BaseModel): - schema = schema.model_json_schema() - self.schema = schema - self.tools = _wrap_tools(tools or []) - self.tool_results = tool_results or [] - self.options = options or {} - # Explicit messages= list, if the caller supplied one. Copied so - # later mutation by the caller doesn't alter the Prompt. - self._explicit_messages = list(messages) if messages is not None else None - - @property - def prompt(self): - "The text of the prompt, with any fragments concatenated." - return "\n".join(self.fragments + ([self._prompt] if self._prompt else [])) - - @property - def system(self): - "The system prompt, with any system fragments concatenated." - bits = [ - bit.strip() - for bit in (self.system_fragments + [self._system or ""]) - if bit.strip() - ] - return "\n\n".join(bits) - - @property - def messages(self): - """Canonical list of Message objects for this prompt. - - **Invariant:** this property returns exactly what the model - was (or will be) sent for this turn — the full chain including - any prior conversation history. - - - If ``messages=`` was passed explicitly, it is authoritative: - returned verbatim. Other kwargs (``prompt=``, ``system=``, - ``attachments=``, ``tool_results=``) are ignored for the - messages list (they remain available via ``prompt.prompt``, - ``prompt.system``, etc., for adapters that still read them). - - Otherwise the list is synthesized from the legacy kwargs - (system, tool_results, prompt, attachments), producing just - the current turn — prior history is not folded in, because - no conversation context is reachable here. - - Conversation.prompt / AsyncConversation.prompt / reply() all - pre-compute the full chain and pass it as ``messages=``, so - ``response.prompt.messages`` after those paths is the full - chain. - """ - from .parts import ( - AttachmentPart, - Message, - TextPart, - ToolResultPart, - ) - - if self._explicit_messages is not None: - return list(self._explicit_messages) - - result: List["Message"] = [] - - if self.system: - result.append( - Message(role="system", parts=[TextPart(text=self.system)]) - ) - - if self.tool_results: - result.append( - Message( - role="tool", - parts=[ - ToolResultPart( - name=tr.name, - output=tr.output, - tool_call_id=tr.tool_call_id, - ) - for tr in self.tool_results - ], - ) - ) - - user_parts: List[Any] = [] - if self.prompt: - user_parts.append(TextPart(text=self.prompt)) - for att in self.attachments: - user_parts.append(AttachmentPart(attachment=att)) - if user_parts: - result.append(Message(role="user", parts=user_parts)) - - return result - - -def _wrap_tools(tools: List[ToolDef]) -> List[Tool]: - wrapped_tools = [] - for tool in tools: - if isinstance(tool, Tool): - wrapped_tools.append(tool) - elif isinstance(tool, Toolbox): - wrapped_tools.extend(tool.tools()) - elif callable(tool): - wrapped_tools.append(Tool.function(tool)) - else: - raise ValueError(f"Invalid tool: {tool}") - return wrapped_tools - - -@dataclass -class _BaseConversation: - model: "_BaseModel" - id: str = field(default_factory=lambda: str(monotonic_ulid()).lower()) - name: Optional[str] = None - responses: List["_BaseResponse"] = field(default_factory=list) - tools: Optional[List[ToolDef]] = None - chain_limit: Optional[int] = None - - @classmethod - @abstractmethod - def from_row(cls, row: Any) -> "_BaseConversation": - raise NotImplementedError - - def _build_full_chain( - self, - prompt: Optional[str], - attachments, - tool_results, - explicit_messages, - ) -> List[Any]: - """Build the full message chain for the next turn. - - Walks this conversation's responses to collect prior history, - then appends the new turn's content (explicit messages first, - or synthesized from prompt/attachments/tool_results). - - Returns the list that should be passed as ``messages=`` to the - Prompt constructor so that ``response.prompt.messages`` equals - exactly what the model sees. - - If ``explicit_messages`` is provided, the caller has opted out - of history walking — the list is used as-is. - """ - from .parts import ( - AttachmentPart, - Message, - TextPart, - ToolResultPart, - ) - - if explicit_messages is not None: - return list(explicit_messages) - - chain: List[Any] = [] - for prev in self.responses: - # prev.prompt.messages already contains prev's full input - # chain under the new invariant, but for the FIRST hop into - # a conversation we defensively de-duplicate by only - # concatenating the last response's full chain (which - # transitively includes everything before it). - pass - if self.responses: - last = self.responses[-1] - chain.extend(last.prompt.messages) - # Append that response's own output (structured messages). - try: - chain.extend(last.messages) - except ValueError: - # AsyncResponse not yet awaited — the caller shouldn't - # be constructing a next turn without awaiting first. - pass - - # Append the new turn's input - if tool_results: - chain.append( - Message( - role="tool", - parts=[ - ToolResultPart( - name=tr.name, - output=tr.output, - tool_call_id=tr.tool_call_id, - ) - for tr in tool_results - ], - ) - ) - - user_parts: List[Any] = [] - if prompt: - user_parts.append(TextPart(text=prompt)) - for att in attachments or []: - user_parts.append(AttachmentPart(attachment=att)) - if user_parts: - chain.append(Message(role="user", parts=user_parts)) - - return chain - - -@dataclass -class Conversation(_BaseConversation): - before_call: Optional[BeforeCallSync] = None - after_call: Optional[AfterCallSync] = None - - def prompt( - self, - prompt: Optional[str] = None, - *, - fragments: Optional[List[Union[str, Fragment]]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - system_fragments: Optional[List[Union[str, Fragment]]] = None, - messages: Optional[List[Any]] = None, - stream: bool = True, - key: Optional[str] = None, - **options, - ) -> "Response": - # Build the authoritative chain so response.prompt.messages - # equals exactly what the model sees for this turn. - chain = self._build_full_chain( - prompt=prompt, - attachments=attachments, - tool_results=tool_results, - explicit_messages=messages, - ) - return Response( - Prompt( - prompt, - model=self.model, - fragments=fragments, - attachments=attachments, - system=system, - schema=schema, - tools=tools or self.tools, - tool_results=tool_results, - system_fragments=system_fragments, - messages=chain, - options=self.model.Options(**options), - ), - self.model, - stream, - conversation=self, - key=key, - ) - - def chain( - self, - prompt: Optional[str] = None, - *, - fragments: Optional[List[str]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - system_fragments: Optional[List[str]] = None, - messages: Optional[List[Any]] = None, - stream: bool = True, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - chain_limit: Optional[int] = None, - before_call: Optional[BeforeCallSync] = None, - after_call: Optional[AfterCallSync] = None, - key: Optional[str] = None, - options: Optional[dict] = None, - ) -> "ChainResponse": - self.model._validate_attachments(attachments) - # Parity with Conversation.prompt: pre-bake the full chain so - # response.prompt.messages is authoritative for the first turn - # of the chain loop. Subsequent tool-result turns extend the - # chain via _chain_for_tool_results. - chain_messages = self._build_full_chain( - prompt=prompt, - attachments=attachments, - tool_results=tool_results, - explicit_messages=messages, - ) - return ChainResponse( - Prompt( - prompt, - fragments=fragments, - attachments=attachments, - system=system, - schema=schema, - tools=tools or self.tools, - tool_results=tool_results, - system_fragments=system_fragments, - messages=chain_messages, - model=self.model, - options=self.model.Options(**(options or {})), - ), - model=self.model, - stream=stream, - conversation=self, - key=key, - before_call=before_call or self.before_call, - after_call=after_call or self.after_call, - chain_limit=chain_limit if chain_limit is not None else self.chain_limit, - ) - - @classmethod - def from_row(cls, row): - from llm import get_model - - return cls( - model=get_model(row["model"]), - id=row["id"], - name=row["name"], - ) - - def __repr__(self): - count = len(self.responses) - s = "s" if count == 1 else "" - return f"<{self.__class__.__name__}: {self.id} - {count} response{s}" - - -@dataclass -class AsyncConversation(_BaseConversation): - before_call: Optional[BeforeCallAsync] = None - after_call: Optional[AfterCallAsync] = None - - def chain( - self, - prompt: Optional[str] = None, - *, - fragments: Optional[List[str]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - system_fragments: Optional[List[str]] = None, - messages: Optional[List[Any]] = None, - stream: bool = True, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - chain_limit: Optional[int] = None, - before_call: Optional[BeforeCallAsync] = None, - after_call: Optional[AfterCallAsync] = None, - key: Optional[str] = None, - options: Optional[dict] = None, - ) -> "AsyncChainResponse": - self.model._validate_attachments(attachments) - chain_messages = self._build_full_chain( - prompt=prompt, - attachments=attachments, - tool_results=tool_results, - explicit_messages=messages, - ) - return AsyncChainResponse( - Prompt( - prompt, - fragments=fragments, - attachments=attachments, - system=system, - schema=schema, - tools=tools or self.tools, - tool_results=tool_results, - system_fragments=system_fragments, - messages=chain_messages, - model=self.model, - options=self.model.Options(**(options or {})), - ), - model=self.model, - stream=stream, - conversation=self, - key=key, - before_call=before_call or self.before_call, - after_call=after_call or self.after_call, - chain_limit=chain_limit if chain_limit is not None else self.chain_limit, - ) - - def prompt( - self, - prompt: Optional[str] = None, - *, - fragments: Optional[List[str]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - system_fragments: Optional[List[str]] = None, - messages: Optional[List[Any]] = None, - stream: bool = True, - key: Optional[str] = None, - **options, - ) -> "AsyncResponse": - chain = self._build_full_chain( - prompt=prompt, - attachments=attachments, - tool_results=tool_results, - explicit_messages=messages, - ) - return AsyncResponse( - Prompt( - prompt, - model=self.model, - fragments=fragments, - attachments=attachments, - system=system, - schema=schema, - tools=tools, - tool_results=tool_results, - system_fragments=system_fragments, - messages=chain, - options=self.model.Options(**options), - ), - self.model, - stream, - conversation=self, - key=key, - ) - - def to_sync_conversation(self): - return Conversation( - model=self.model, - id=self.id, - name=self.name, - responses=[], # Because we only use this in logging - tools=self.tools, - chain_limit=self.chain_limit, - ) - - @classmethod - def from_row(cls, row): - from llm import get_async_model - - return cls( - model=get_async_model(row["model"]), - id=row["id"], - name=row["name"], - ) - - def __repr__(self): - count = len(self.responses) - s = "s" if count == 1 else "" - return f"<{self.__class__.__name__}: {self.id} - {count} response{s}" - - -FRAGMENT_SQL = """ -select - 'prompt' as fragment_type, - fragments.content, - pf."order" as ord -from prompt_fragments pf -join fragments on pf.fragment_id = fragments.id -where pf.response_id = :response_id -union all -select - 'system' as fragment_type, - fragments.content, - sf."order" as ord -from system_fragments sf -join fragments on sf.fragment_id = fragments.id -where sf.response_id = :response_id -order by fragment_type desc, ord asc; -""" - - -class _BaseResponse: - """Base response class shared between sync and async responses""" - - id: str - prompt: "Prompt" - stream: bool - resolved_model: Optional[str] = None - conversation: Optional["_BaseConversation"] = None - _key: Optional[str] = None - _tool_calls: List[ToolCall] = [] - - def __init__( - self, - prompt: Prompt, - model: "_BaseModel", - stream: bool, - conversation: Optional[_BaseConversation] = None, - key: Optional[str] = None, - ): - self.id = str(monotonic_ulid()).lower() - self.prompt = prompt - self._prompt_json = None - self.model = model - self.stream = stream - self._key = key - self._chunks: List[str] = [] - # Every StreamEvent ever yielded by execute(), in order. Plain - # str yields are wrapped as StreamEvent(type="text", part_index=0) - # so this buffer is the single source of truth for replay and - # for assembling response.messages. - self._stream_events: List[Any] = [] - # Plugins set this when the provider reports an opaque reasoning - # token count (no streamed reasoning text). _build_parts() - # prepends a ReasoningPart(redacted=True, token_count=N) when - # non-zero. - self._reasoning_token_count: int = 0 - self._done = False - self._tool_calls: List[ToolCall] = [] - self.response_json: Optional[Dict[str, Any]] = None - self.conversation = conversation - self.attachments: List[Attachment] = [] - self._start: Optional[float] = None - self._end: Optional[float] = None - self._start_utcnow: Optional[datetime.datetime] = None - self.input_tokens: Optional[int] = None - self.output_tokens: Optional[int] = None - self.token_details: Optional[dict] = None - self.done_callbacks: List[Callable] = [] - - if self.prompt.schema and not self.model.supports_schema: - raise ValueError(f"{self.model} does not support schemas") - - if self.prompt.tools and not self.model.supports_tools: - raise ValueError(f"{self.model} does not support tools") - - def _process_chunk(self, chunk): - """Normalize a chunk from execute() into a StreamEvent and return - the text str (or None) that __iter__ should yield. - - Plain str yields from legacy plugins are wrapped as text events - at part_index=0. Side effects: populates self._stream_events and - self._chunks. - """ - from .parts import StreamEvent - - if isinstance(chunk, StreamEvent): - self._stream_events.append(chunk) - if chunk.type == "text": - self._chunks.append(chunk.chunk) - return chunk.chunk - return None - # Legacy plain-str plugin. - event = StreamEvent(type="text", chunk=chunk, part_index=0) - self._stream_events.append(event) - self._chunks.append(chunk) - return chunk - - def _build_parts(self) -> List[Any]: - """Assemble Part objects from the accumulated stream events. - - Events sharing a part_index group into one Part. Mixing - families (text vs tool_call vs reasoning vs tool_result) at the - same index is a plugin bug — raises ValueError instead of - silently dropping content. - - Fallback: when no stream events were recorded (response was - rehydrated from SQLite via ``from_row``), synthesize a - TextPart from ``self._chunks`` plus any ``self._tool_calls`` - restored by the row loader. Reasoning signatures are not - recoverable from SQLite in this fallback — use - ``response.to_dict()`` / ``Response.from_dict()`` for - structure-preserving persistence. - """ - from .parts import ( - ReasoningPart, - TextPart, - ToolCallPart, - ToolResultPart, - ) - - if not self._stream_events: - # Rehydrated-from-SQLite path: assemble from _chunks + - # _tool_calls so response.messages isn't empty after - # from_row, and Conversation.prompt-built chains include - # the assistant turn on follow-up calls. - parts: List[Any] = [] - text = "".join(self._chunks) - if text: - parts.append(TextPart(text=text)) - for tc in self._tool_calls: - parts.append( - ToolCallPart( - name=tc.name, - arguments=tc.arguments or {}, - tool_call_id=tc.tool_call_id, - ) - ) - reasoning_token_count = getattr( - self, "_reasoning_token_count", 0 - ) - if reasoning_token_count: - parts.insert( - 0, - ReasoningPart( - text="", - redacted=True, - token_count=reasoning_token_count, - ), - ) - return parts - - def family(t: str) -> str: - if t in ("tool_call_name", "tool_call_args"): - return "tool_call" - return t - - parts: List[Any] = [] - current_index: Optional[int] = None - current_family: Optional[str] = None - text_buf: List[str] = [] - tool_name: Optional[str] = None - tool_args_buf: List[str] = [] - tool_call_id: Optional[str] = None - server_executed = False - tool_result_name: Optional[str] = None - pm_merged: Optional[Dict[str, Any]] = None - - def finalize(): - nonlocal pm_merged - if current_family is None: - return - if current_family == "text": - text = "".join(text_buf) - if text: - parts.append(TextPart(text=text, provider_metadata=pm_merged)) - elif current_family == "reasoning": - text = "".join(text_buf) - if text: - parts.append( - ReasoningPart(text=text, provider_metadata=pm_merged) - ) - elif current_family == "tool_call": - args_str = "".join(tool_args_buf) - try: - arguments = json.loads(args_str) if args_str else {} - except json.JSONDecodeError: - arguments = {"_raw": args_str} - parts.append( - ToolCallPart( - name=tool_name or "", - arguments=arguments, - tool_call_id=tool_call_id, - server_executed=server_executed, - provider_metadata=pm_merged, - ) - ) - elif current_family == "tool_result": - parts.append( - ToolResultPart( - name=tool_result_name or "", - output="".join(text_buf), - tool_call_id=tool_call_id, - server_executed=server_executed, - provider_metadata=pm_merged, - ) - ) - - for event in self._stream_events: - ev_family = family(event.type) - if event.part_index != current_index: - finalize() - current_index = event.part_index - current_family = ev_family - text_buf = [] - tool_name = None - tool_args_buf = [] - tool_call_id = None - server_executed = False - tool_result_name = None - pm_merged = None - elif current_family is not None and ev_family != current_family: - raise ValueError( - f"StreamEvent type {event.type!r} is incompatible with " - f"prior type at part_index={event.part_index}. " - "Allocate a new part_index for a different content type." - ) - - if event.type == "text": - text_buf.append(event.chunk) - elif event.type == "reasoning": - text_buf.append(event.chunk) - elif event.type == "tool_call_name": - tool_name = (tool_name or "") + event.chunk - if event.tool_call_id: - tool_call_id = event.tool_call_id - if event.server_executed: - server_executed = True - elif event.type == "tool_call_args": - tool_args_buf.append(event.chunk) - if event.tool_call_id and tool_call_id is None: - tool_call_id = event.tool_call_id - if event.server_executed: - server_executed = True - elif event.type == "tool_result": - text_buf.append(event.chunk) - if event.tool_call_id and tool_call_id is None: - tool_call_id = event.tool_call_id - if event.server_executed: - server_executed = True - if event.tool_name: - tool_result_name = event.tool_name - - if event.provider_metadata: - merged = dict(pm_merged) if pm_merged else {} - for k, v in event.provider_metadata.items(): - merged[k] = v - pm_merged = merged - - finalize() - - if self._reasoning_token_count: - parts.insert( - 0, - ReasoningPart( - text="", - redacted=True, - token_count=self._reasoning_token_count, - ), - ) - - return parts - - def add_tool_call(self, tool_call: ToolCall): - self._tool_calls.append(tool_call) - - def set_usage( - self, - *, - input: Optional[int] = None, - output: Optional[int] = None, - details: Optional[dict] = None, - ): - self.input_tokens = input - self.output_tokens = output - self.token_details = details - - def set_resolved_model(self, model_id: str): - self.resolved_model = model_id - - @classmethod - def from_row(cls, db, row, _async=False): - from llm import get_model, get_async_model - - if _async: - model = get_async_model(row["model"]) - else: - model = get_model(row["model"]) - - # Schema - schema = None - if row["schema_id"]: - schema = json.loads(db["schemas"].get(row["schema_id"])["content"]) - - # Tool definitions and results for prompt - tools = [ - Tool( - name=tool_row["name"], - description=tool_row["description"], - input_schema=json.loads(tool_row["input_schema"]), - # In this case we don't have a reference to the actual Python code - # but that's OK, we should not need it for prompts deserialized from DB - implementation=None, - plugin=tool_row["plugin"], - ) - for tool_row in db.query( - """ - select tools.* from tools - join tool_responses on tools.id = tool_responses.tool_id - where tool_responses.response_id = ? - """, - [row["id"]], - ) - ] - tool_results = [ - ToolResult( - name=tool_results_row["name"], - output=tool_results_row["output"], - tool_call_id=tool_results_row["tool_call_id"], - ) - for tool_results_row in db.query( - """ - select * from tool_results - where response_id = ? - """, - [row["id"]], - ) - ] - - all_fragments = list(db.query(FRAGMENT_SQL, {"response_id": row["id"]})) - fragments = [ - row["content"] for row in all_fragments if row["fragment_type"] == "prompt" - ] - system_fragments = [ - row["content"] for row in all_fragments if row["fragment_type"] == "system" - ] - response = cls( - model=model, - prompt=Prompt( - prompt=row["prompt"], - model=model, - fragments=fragments, - attachments=[], - system=row["system"], - schema=schema, - tools=tools, - tool_results=tool_results, - system_fragments=system_fragments, - options=model.Options(**json.loads(row["options_json"])), - ), - stream=False, - ) - prompt_json = json.loads(row["prompt_json"] or "null") - response.id = row["id"] - response._prompt_json = prompt_json - response.response_json = json.loads(row["response_json"] or "null") - response._done = True - response._chunks = [row["response"]] - # Attachments - response.attachments = [ - Attachment.from_row(attachment_row) - for attachment_row in db.query( - """ - select attachments.* from attachments - join prompt_attachments on attachments.id = prompt_attachments.attachment_id - where prompt_attachments.response_id = ? - order by prompt_attachments."order" - """, - [row["id"]], - ) - ] - # Tool calls - response._tool_calls = [ - ToolCall( - name=tool_row["name"], - arguments=json.loads(tool_row["arguments"]), - tool_call_id=tool_row["tool_call_id"], - ) - for tool_row in db.query( - """ - select * from tool_calls - where response_id = ? - order by tool_call_id - """, - [row["id"]], - ) - ] - - return response - - def token_usage(self) -> str: - return token_usage_string( - self.input_tokens, self.output_tokens, self.token_details - ) - - def log_to_db(self, db): - conversation = self.conversation - if not conversation: - conversation = Conversation(model=self.model) - db["conversations"].insert( - { - "id": conversation.id, - "name": _conversation_name( - self.prompt.prompt or self.prompt.system or "" - ), - "model": conversation.model.model_id, - }, - ignore=True, - ) - schema_id = None - if self.prompt.schema: - schema_id, schema_json = make_schema_id(self.prompt.schema) - db["schemas"].insert({"id": schema_id, "content": schema_json}, ignore=True) - - response_id = self.id - replacements = {} - # Include replacements from previous responses - for previous_response in conversation.responses[:-1]: - for fragment in (previous_response.prompt.fragments or []) + ( - previous_response.prompt.system_fragments or [] - ): - fragment_id = ensure_fragment(db, fragment) - replacements[f"f:{fragment_id}"] = fragment - replacements[f"r:{previous_response.id}"] = ( - previous_response.text_or_raise() - ) - - for i, fragment in enumerate(self.prompt.fragments): - fragment_id = ensure_fragment(db, fragment) - replacements[f"f{fragment_id}"] = fragment - db["prompt_fragments"].insert( - { - "response_id": response_id, - "fragment_id": fragment_id, - "order": i, - }, - ) - for i, fragment in enumerate(self.prompt.system_fragments): - fragment_id = ensure_fragment(db, fragment) - replacements[f"f{fragment_id}"] = fragment - db["system_fragments"].insert( - { - "response_id": response_id, - "fragment_id": fragment_id, - "order": i, - }, - ) - - response_text = self.text_or_raise() - replacements[f"r:{response_id}"] = response_text - json_data = self.json() - - response = { - "id": response_id, - "model": self.model.model_id, - "prompt": self.prompt._prompt, - "system": self.prompt._system, - "prompt_json": condense_json(self._prompt_json, replacements), - "options_json": { - key: value - for key, value in dict(self.prompt.options).items() - if value is not None - }, - "response": response_text, - "response_json": condense_json(json_data, replacements), - "conversation_id": conversation.id, - "duration_ms": self.duration_ms(), - "datetime_utc": self.datetime_utc(), - "input_tokens": self.input_tokens, - "output_tokens": self.output_tokens, - "token_details": ( - json.dumps(self.token_details) if self.token_details else None - ), - "schema_id": schema_id, - "resolved_model": self.resolved_model, - } - db["responses"].insert(response) - - # Persist any attachments - loop through with index - for index, attachment in enumerate(self.prompt.attachments): - attachment_id = attachment.id() - db["attachments"].insert( - { - "id": attachment_id, - "type": attachment.resolve_type(), - "path": attachment.path, - "url": attachment.url, - "content": attachment.content, - }, - replace=True, - ) - db["prompt_attachments"].insert( - { - "response_id": response_id, - "attachment_id": attachment_id, - "order": index, - }, - ) - - # Persist any tools, tool calls and tool results - tool_ids_by_name = {} - for tool in self.prompt.tools: - tool_id = ensure_tool(db, tool) - tool_ids_by_name[tool.name] = tool_id - db["tool_responses"].insert( - { - "tool_id": tool_id, - "response_id": response_id, - } - ) - for tool_call in self.tool_calls(): # TODO Should be _or_raise() - db["tool_calls"].insert( - { - "response_id": response_id, - "tool_id": tool_ids_by_name.get(tool_call.name) or None, - "name": tool_call.name, - "arguments": json.dumps(tool_call.arguments), - "tool_call_id": tool_call.tool_call_id, - } - ) - for tool_result in self.prompt.tool_results: - instance_id = None - if tool_result.instance: - try: - if not tool_result.instance.instance_id: - tool_result.instance.instance_id = ( - db["tool_instances"] - .insert( - { - "plugin": tool.plugin, - "name": tool.name.split("_")[0], - "arguments": json.dumps( - tool_result.instance._config - ), - } - ) - .last_pk - ) - instance_id = tool_result.instance.instance_id - except AttributeError: - pass - tool_result_id = ( - db["tool_results"] - .insert( - { - "response_id": response_id, - "tool_id": tool_ids_by_name.get(tool_result.name) or None, - "name": tool_result.name, - "output": tool_result.output, - "tool_call_id": tool_result.tool_call_id, - "instance_id": instance_id, - "exception": ( - ( - "{}: {}".format( - tool_result.exception.__class__.__name__, - str(tool_result.exception), - ) - ) - if tool_result.exception - else None - ), - } - ) - .last_pk - ) - # Persist attachments for tool results - for index, attachment in enumerate(tool_result.attachments): - attachment_id = attachment.id() - db["attachments"].insert( - { - "id": attachment_id, - "type": attachment.resolve_type(), - "path": attachment.path, - "url": attachment.url, - "content": attachment.content, - }, - replace=True, - ) - db["tool_results_attachments"].insert( - { - "tool_result_id": tool_result_id, - "attachment_id": attachment_id, - "order": index, - }, - ) - - -def _response_to_dict(response: "_BaseResponse") -> ResponseDict: - """Shared serializer for Response.to_dict / AsyncResponse.to_dict. - - The output is a JSON-safe dict — store it anywhere (file, Redis, - Postgres, HTTP body) and round-trip via Response.from_dict. - """ - options = { - key: value - for key, value in dict(response.prompt.options).items() - if value is not None - } - payload: Dict[str, Any] = { - "model": response.model.model_id, - "prompt": { - "messages": [m.to_dict() for m in response.prompt.messages], - }, - "messages": [m.to_dict() for m in response.messages], - } - if options: - payload["prompt"]["options"] = options - if response.prompt._system: - payload["prompt"]["system"] = response.prompt._system - # Optional audit fields — helpful for debugging, not needed for reply(). - if response.id: - payload["id"] = response.id - if response._done: - if response.input_tokens is not None or response.output_tokens is not None: - usage: Dict[str, Any] = {} - if response.input_tokens is not None: - usage["input"] = response.input_tokens - if response.output_tokens is not None: - usage["output"] = response.output_tokens - if response.token_details is not None: - usage["details"] = response.token_details - payload["usage"] = usage - if response._start_utcnow is not None: - payload["datetime_utc"] = response._start_utcnow.isoformat() - return payload - - -def _response_from_dict( - data: Dict[str, Any], - cls, - *, - model=None, - async_: bool = False, -) -> "_BaseResponse": - """Shared deserializer for Response.from_dict / AsyncResponse.from_dict.""" - from .parts import Message - - if model is None: - from llm import get_async_model, get_model - - getter = get_async_model if async_ else get_model - model = getter(data["model"]) - - prompt_data = data.get("prompt", {}) - input_messages = [ - Message.from_dict(m) for m in prompt_data.get("messages", []) - ] - output_messages = [ - Message.from_dict(m) for m in data.get("messages", []) - ] - - options_kwargs = prompt_data.get("options") or {} - system = prompt_data.get("system") - - prompt = Prompt( - None, - model=model, - messages=input_messages, - system=system, - options=model.Options(**options_kwargs), - ) - response = cls(prompt, model=model, stream=False) - # Preserve id for audit continuity. - if "id" in data: - response.id = data["id"] - # Rebuild chunks from the assistant's text parts so response.text() - # works without re-running the assembler. - from .parts import TextPart - - response._chunks = [ - p.text - for m in output_messages - for p in m.parts - if isinstance(p, TextPart) and p.text - ] - # Stash the structured output so response.messages returns the - # full picture (reasoning, tool calls, signatures) without needing - # a StreamEvent replay. - response._loaded_messages = output_messages - response._done = True - # Restore usage if present. - usage = data.get("usage") - if usage: - response.input_tokens = usage.get("input") - response.output_tokens = usage.get("output") - response.token_details = usage.get("details") - return response - - -class Response(_BaseResponse): - "Sync response from a model." - - model: "Model" - conversation: Optional["Conversation"] = None - - def reply( - self, - prompt: Optional[str] = None, - *, - messages: Optional[List[Any]] = None, - **kwargs, - ) -> "Response": - """Continue the conversation from this response. - - Builds the next turn's chain as - ``self.prompt.messages + self.messages + [user(prompt)]`` and - calls ``self.model.prompt(messages=chain, ...)``. No - Conversation object required — the Response carries everything - needed. - - If ``messages=`` is passed, its contents are appended to the - chain instead of (or in addition to) the ``prompt`` string. - """ - from .parts import Message, TextPart - - self._force() - chain: List[Any] = list(self.prompt.messages) + list(self.messages) - if prompt: - chain.append( - Message(role="user", parts=[TextPart(text=prompt)]) - ) - if messages: - chain.extend(messages) - return self.model.prompt(messages=chain, **kwargs) - - def to_dict(self) -> ResponseDict: - """Serialize this response for JSON persistence. - - Captures exactly what is needed to continue the conversation: - model id, the input chain that was sent - (``response.prompt.messages``), the structured assistant output - (``response.messages``), and any explicit options. Pair with - :meth:`Response.from_dict` to rehydrate and - :meth:`Response.reply` to continue. - - Returns :class:`~llm.serialization.ResponseDict`. - """ - return _response_to_dict(self) - - @classmethod - def from_dict( - cls, - data: ResponseDict, - *, - model: Optional["Model"] = None, - ) -> "Response": - """Rehydrate a Response from a ``to_dict()`` payload. - - The returned Response is in the ``_done`` state with - ``response.text()`` and ``response.messages`` populated. - ``model`` overrides the stored model id (useful for continuing - on a different model). - """ - return _response_from_dict(data, cls, model=model, async_=False) - - def on_done(self, callback): - "Register a callback to be called when the response is complete." - if not self._done: - self.done_callbacks.append(callback) - else: - callback(self) - - def _on_done(self): - for callback in self.done_callbacks: - callback(self) - - def __str__(self) -> str: - return self.text() - - def _force(self): - if not self._done: - list(self) - - def text(self) -> str: - "Return the full text of the response, executing the prompt if needed." - self._force() - return "".join(self._chunks) - - def text_or_raise(self) -> str: - return self.text() - - def execute_tool_calls( - self, - *, - before_call: Optional[BeforeCallSync] = None, - after_call: Optional[AfterCallSync] = None, - ) -> List[ToolResult]: - tool_results = [] - tools_by_name = {tool.name: tool for tool in self.prompt.tools} - - # Run prepare() on all Toolbox instances that need it - instances_to_prepare: list[Toolbox] = [] - for tool_to_prep in tools_by_name.values(): - inst = _get_instance(tool_to_prep.implementation) - if isinstance(inst, Toolbox) and not getattr(inst, "_prepared", False): - instances_to_prepare.append(inst) - - for inst in instances_to_prepare: - inst.prepare() - inst._prepared = True - - for tool_call in self.tool_calls(): - tool: Optional[Tool] = tools_by_name.get(tool_call.name) - # Tool could be None if the tool was not found in the prompt tools, - # but we still call the before_call method: - if before_call: - try: - cb_result = before_call(tool, tool_call) - if inspect.isawaitable(cb_result): - raise TypeError( - "Asynchronous 'before_call' callback provided to a synchronous tool execution context. " - "Please use an async chain/response or a synchronous callback." - ) - except CancelToolCall as ex: - tool_results.append( - ToolResult( - name=tool_call.name, - output="Cancelled: " + str(ex), - tool_call_id=tool_call.tool_call_id, - exception=ex, - ) - ) - continue - - if tool is None: - msg = 'tool "{}" does not exist'.format(tool_call.name) - tool_results.append( - ToolResult( - name=tool_call.name, - output="Error: " + msg, - tool_call_id=tool_call.tool_call_id, - exception=KeyError(msg), - ) - ) - continue - - if not tool.implementation: - raise ValueError( - "No implementation available for tool: {}".format(tool_call.name) - ) - - attachments = [] - exception = None - - try: - if inspect.iscoroutinefunction(tool.implementation): - result = asyncio.run(tool.implementation(**tool_call.arguments)) - else: - result = tool.implementation(**tool_call.arguments) - - if isinstance(result, ToolOutput): - attachments = result.attachments - result = result.output - - if not isinstance(result, str): - result = json.dumps(result, default=repr) - except Exception as ex: - result = f"Error: {ex}" - exception = ex - - tool_result_obj = ToolResult( - name=tool_call.name, - output=result, - attachments=attachments, - tool_call_id=tool_call.tool_call_id, - instance=_get_instance(tool.implementation), - exception=exception, - ) - - if after_call: - cb_result = after_call(tool, tool_call, tool_result_obj) - if inspect.isawaitable(cb_result): - raise TypeError( - "Asynchronous 'after_call' callback provided to a synchronous tool execution context. " - "Please use an async chain/response or a synchronous callback." - ) - tool_results.append(tool_result_obj) - return tool_results - - def tool_calls(self) -> List[ToolCall]: - "Return the list of tool calls made during this response." - self._force() - return self._tool_calls - - def tool_calls_or_raise(self) -> List[ToolCall]: - return self.tool_calls() - - def json(self) -> Optional[Dict[str, Any]]: - "Return the raw JSON response from the model, if available." - self._force() - return self.response_json - - def duration_ms(self) -> int: - self._force() - return int(((self._end or 0) - (self._start or 0)) * 1000) - - def datetime_utc(self) -> str: - self._force() - return self._start_utcnow.isoformat() if self._start_utcnow else "" - - def usage(self) -> Usage: - "Return token usage information for this response." - self._force() - return Usage( - input=self.input_tokens, - output=self.output_tokens, - details=self.token_details, - ) - - def _iter_events(self): - """Drive self.model.execute() once. Yields every chunk it - produces, each already appended to self._stream_events by - _process_chunk as a side effect. - """ - if isinstance(self.model, Model): - generator = self.model.execute( - self.prompt, - stream=self.stream, - response=self, - conversation=self.conversation, - ) - elif isinstance(self.model, KeyModel): - generator = self.model.execute( - self.prompt, - stream=self.stream, - response=self, - conversation=self.conversation, - key=self.model.get_key(self._key), - ) - else: - raise Exception("self.model must be a Model or KeyModel") - - for chunk in generator: - assert chunk is not None - yield chunk - - def __iter__(self) -> Iterator[str]: - self._start = time.monotonic() - self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) - if self._done: - yield from self._chunks - return - - for chunk in self._iter_events(): - text = self._process_chunk(chunk) - if text is not None: - yield text - - if self.conversation: - self.conversation.responses.append(self) - self._end = time.monotonic() - self._done = True - self._on_done() - - def stream_events(self): - """Yield StreamEvent objects as the model produces them. - - Whichever of __iter__ and stream_events runs first during live - streaming consumes the underlying generator. After completion, - both work — each replays from its own buffer. - """ - if self._done: - yield from self._stream_events - return - - self._start = time.monotonic() - self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) - for chunk in self._iter_events(): - # _process_chunk appends to self._stream_events; use it as - # the canonical source for what to yield so the replay path - # matches the live path byte-for-byte. - self._process_chunk(chunk) - yield self._stream_events[-1] - - if self.conversation: - self.conversation.responses.append(self) - self._end = time.monotonic() - self._done = True - self._on_done() - - @property - def messages(self) -> List[Any]: - """List of Message objects produced by this response. - - Almost always a single assistant Message; multiple messages are - possible for providers that emit multi-message responses during - server-side tool execution (not in this phase's scope). - - Responses rehydrated via ``Response.from_dict`` short-circuit - and return the stored messages directly. - """ - from .parts import Message - - loaded = getattr(self, "_loaded_messages", None) - if loaded is not None: - return list(loaded) - self._force() - parts = self._build_parts() - if not parts: - return [] - return [Message(role="assistant", parts=parts)] - - def __repr__(self): - text = "... not yet done ..." - if self._done: - text = "".join(self._chunks) - return "".format(self.prompt.prompt, text) - - -class AsyncResponse(_BaseResponse): - "Async response from a model." - - model: "AsyncModel" - conversation: Optional["AsyncConversation"] = None - - def reply( - self, - prompt: Optional[str] = None, - *, - messages: Optional[List[Any]] = None, - **kwargs, - ) -> "AsyncResponse": - """Async counterpart of Response.reply(). Requires this response - to have been awaited (so self.messages is available). - """ - from .parts import Message, TextPart - - if not self._done: - raise ValueError( - "Response not yet awaited — call `await response` before reply()" - ) - chain: List[Any] = list(self.prompt.messages) + list(self.messages) - if prompt: - chain.append( - Message(role="user", parts=[TextPart(text=prompt)]) - ) - if messages: - chain.extend(messages) - return self.model.prompt(messages=chain, **kwargs) - - def to_dict(self) -> ResponseDict: - """Async counterpart of Response.to_dict(). Requires awaiting.""" - if not self._done: - raise ValueError( - "Response not yet awaited — call `await response` before to_dict()" - ) - return _response_to_dict(self) - - @classmethod - def from_dict( - cls, - data: ResponseDict, - *, - model: Optional["AsyncModel"] = None, - ) -> "AsyncResponse": - """Async counterpart of Response.from_dict().""" - return _response_from_dict(data, cls, model=model, async_=True) - - @classmethod - def from_row(cls, db, row, _async=False): - return super().from_row(db, row, _async=True) - - async def on_done(self, callback): - "Register a callback to be called when the response is complete." - if not self._done: - self.done_callbacks.append(callback) - else: - if callable(callback): - # Ensure we handle both sync and async callbacks correctly - processed_callback = callback(self) - if inspect.isawaitable(processed_callback): - await processed_callback - elif inspect.isawaitable(callback): - await callback - - async def _on_done(self): - for callback_func in self.done_callbacks: - if callable(callback_func): - processed_callback = callback_func(self) - if inspect.isawaitable(processed_callback): - await processed_callback - elif inspect.isawaitable(callback_func): - await callback_func - - async def execute_tool_calls( - self, - *, - before_call: Optional[BeforeCallAsync] = None, - after_call: Optional[AfterCallAsync] = None, - ) -> List[ToolResult]: - tool_calls_list = await self.tool_calls() - tools_by_name = {tool.name: tool for tool in self.prompt.tools} - - # Run async prepare_async() on all Toolbox instances that need it - instances_to_prepare: list[Toolbox] = [] - for tool_to_prep in tools_by_name.values(): - inst = _get_instance(tool_to_prep.implementation) - if isinstance(inst, Toolbox) and not getattr( - inst, "_async_prepared", False - ): - instances_to_prepare.append(inst) - - for inst in instances_to_prepare: - await inst.prepare_async() - inst._async_prepared = True - - indexed_results: List[tuple[int, ToolResult]] = [] - async_tasks: List[asyncio.Task] = [] - - for idx, tc in enumerate(tool_calls_list): - tool: Optional[Tool] = tools_by_name.get(tc.name) - exception: Optional[Exception] = None - - if tool is None: - output = f'Error: tool "{tc.name}" does not exist' - exception = KeyError(tc.name) - elif not tool.implementation: - output = f'Error: tool "{tc.name}" has no implementation' - exception = KeyError(tc.name) - elif inspect.iscoroutinefunction(tool.implementation): - - async def run_async(tc=tc, tool=tool, idx=idx): - # before_call inside the task - if before_call: - try: - cb = before_call(tool, tc) - if inspect.isawaitable(cb): - await cb - except CancelToolCall as ex: - return idx, ToolResult( - name=tc.name, - output="Cancelled: " + str(ex), - tool_call_id=tc.tool_call_id, - exception=ex, - ) - - exception = None - attachments = [] - - try: - result = await tool.implementation(**tc.arguments) - if isinstance(result, ToolOutput): - attachments.extend(result.attachments) - result = result.output - output = ( - result - if isinstance(result, str) - else json.dumps(result, default=repr) - ) - except Exception as ex: - output = f"Error: {ex}" - exception = ex - - tr = ToolResult( - name=tc.name, - output=output, - attachments=attachments, - tool_call_id=tc.tool_call_id, - instance=_get_instance(tool.implementation), - exception=exception, - ) - - # after_call inside the task - if tool is not None and after_call: - cb2 = after_call(tool, tc, tr) - if inspect.isawaitable(cb2): - await cb2 - - return idx, tr - - async_tasks.append(asyncio.create_task(run_async())) - - else: - # Sync implementation: do hooks and call inline - if before_call: - try: - cb = before_call(tool, tc) - if inspect.isawaitable(cb): - await cb - except CancelToolCall as ex: - indexed_results.append( - ( - idx, - ToolResult( - name=tc.name, - output="Cancelled: " + str(ex), - tool_call_id=tc.tool_call_id, - exception=ex, - ), - ) - ) - continue - - exception = None - attachments = [] - - if tool is None: - output = f'Error: tool "{tc.name}" does not exist' - exception = KeyError(tc.name) - else: - try: - res = tool.implementation(**tc.arguments) - if inspect.isawaitable(res): - res = await res - if isinstance(res, ToolOutput): - attachments.extend(res.attachments) - res = res.output - output = ( - res - if isinstance(res, str) - else json.dumps(res, default=repr) - ) - except Exception as ex: - output = f"Error: {ex}" - exception = ex - - tr = ToolResult( - name=tc.name, - output=output, - attachments=attachments, - tool_call_id=tc.tool_call_id, - instance=_get_instance(tool.implementation), - exception=exception, - ) - - if tool is not None and after_call: - cb2 = after_call(tool, tc, tr) - if inspect.isawaitable(cb2): - await cb2 - - indexed_results.append((idx, tr)) - - # Await all async tasks in parallel - if async_tasks: - indexed_results.extend(await asyncio.gather(*async_tasks)) - - # Reorder by original index - indexed_results.sort(key=lambda x: x[0]) - return [tr for _, tr in indexed_results] - - def __aiter__(self): - self._start = time.monotonic() - self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) - if self._done: - self._iter_chunks = list(self._chunks) # Make a copy for iteration - return self - - def _ensure_async_generator(self): - if not hasattr(self, "_generator"): - if isinstance(self.model, AsyncModel): - self._generator = self.model.execute( - self.prompt, - stream=self.stream, - response=self, - conversation=self.conversation, - ) - elif isinstance(self.model, AsyncKeyModel): - self._generator = self.model.execute( - self.prompt, - stream=self.stream, - response=self, - conversation=self.conversation, - key=self.model.get_key(self._key), - ) - else: - raise ValueError("self.model must be an AsyncModel or AsyncKeyModel") - - async def _async_finalize(self): - if self.conversation: - self.conversation.responses.append(self) - self._end = time.monotonic() - self._done = True - if hasattr(self, "_generator"): - del self._generator - await self._on_done() - - async def __anext__(self) -> str: - if self._done: - if hasattr(self, "_iter_chunks") and self._iter_chunks: - return self._iter_chunks.pop(0) - raise StopAsyncIteration - - self._ensure_async_generator() - # Skip non-text events — iteration yields only text. Loop until - # we find a text chunk or the generator is exhausted. - while True: - try: - chunk = await self._generator.__anext__() - except StopAsyncIteration: - await self._async_finalize() - raise - assert chunk is not None - text = self._process_chunk(chunk) - if text is not None: - return text - - async def astream_events(self): - """Yield StreamEvent objects as the model produces them (async).""" - if self._done: - for event in self._stream_events: - yield event - return - - self._start = time.monotonic() - self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) - self._ensure_async_generator() - try: - while True: - try: - chunk = await self._generator.__anext__() - except StopAsyncIteration: - await self._async_finalize() - return - assert chunk is not None - self._process_chunk(chunk) - yield self._stream_events[-1] - finally: - pass - - @property - def messages(self) -> List[Any]: - """List of Message objects produced by this response. - - Raises ValueError if the response has not yet been awaited — - assembly depends on the full event stream. Responses rehydrated - via ``AsyncResponse.from_dict`` short-circuit and return the - stored messages. - """ - from .parts import Message - - loaded = getattr(self, "_loaded_messages", None) - if loaded is not None: - return list(loaded) - if not self._done: - raise ValueError( - "Response not yet awaited — use 'await response' first" - ) - parts = self._build_parts() - if not parts: - return [] - return [Message(role="assistant", parts=parts)] - - async def _force(self): - if not self._done: - temp_chunks = [] - async for chunk in self: - temp_chunks.append(chunk) - # This should populate self._chunks - return self - - def text_or_raise(self) -> str: - if not self._done: - raise ValueError("Response not yet awaited") - return "".join(self._chunks) - - async def text(self) -> str: - "Return the full text of the response, executing the prompt if needed." - await self._force() - return "".join(self._chunks) - - async def tool_calls(self) -> List[ToolCall]: - "Return the list of tool calls made during this response." - await self._force() - return self._tool_calls - - def tool_calls_or_raise(self) -> List[ToolCall]: - if not self._done: - raise ValueError("Response not yet awaited") - return self._tool_calls - - async def json(self) -> Optional[Dict[str, Any]]: - "Return the raw JSON response from the model, if available." - await self._force() - return self.response_json - - async def duration_ms(self) -> int: - await self._force() - return int(((self._end or 0) - (self._start or 0)) * 1000) - - async def datetime_utc(self) -> str: - await self._force() - return self._start_utcnow.isoformat() if self._start_utcnow else "" - - async def usage(self) -> Usage: - "Return token usage information for this response." - await self._force() - return Usage( - input=self.input_tokens, - output=self.output_tokens, - details=self.token_details, - ) - - def __await__(self): - return self._force().__await__() - - async def to_sync_response(self) -> Response: - await self._force() - # This conversion might be tricky if the model is AsyncModel, - # as Response expects a sync Model. For simplicity, we'll assume - # the primary use case is data transfer after completion. - # The model type on the new Response might need careful handling - # if it's intended for further execution. - # For now, let's assume self.model can be cast or is compatible. - sync_model = self.model - if not isinstance(self.model, (Model, KeyModel)): - # This is a placeholder. A proper conversion or shared base might be needed - # if the sync_response needs to be fully functional with its model. - # For now, we pass the async model, which might limit what sync_response can do. - pass - - response = Response( - self.prompt, - sync_model, # This might need adjustment based on how Model/AsyncModel relate - self.stream, - # conversation type needs to be compatible too. - conversation=( - self.conversation.to_sync_conversation() if self.conversation else None - ), - ) - response.id = self.id - response._chunks = list(self._chunks) # Copy chunks - response._done = self._done - response._end = self._end - response._start = self._start - response._start_utcnow = self._start_utcnow - response.input_tokens = self.input_tokens - response.output_tokens = self.output_tokens - response.token_details = self.token_details - response._prompt_json = self._prompt_json - response.response_json = self.response_json - response._tool_calls = list(self._tool_calls) - response.attachments = list(self.attachments) - response.resolved_model = self.resolved_model - return response - - @classmethod - def fake( - cls, - model: "AsyncModel", - prompt: str, - *attachments: List[Attachment], - system: str, - response: str, - ): - "Utility method to help with writing tests" - response_obj = cls( - model=model, - prompt=Prompt( - prompt, - model=model, - attachments=attachments, - system=system, - ), - stream=False, - ) - response_obj._done = True - response_obj._chunks = [response] - return response_obj - - def __repr__(self): - text = "... not yet awaited ..." - if self._done: - text = "".join(self._chunks) - return "".format(self.prompt.prompt, text) - - -def _chain_for_tool_results( - prior_response, tool_results, attachments -) -> List[Any]: - """Build the message chain for a tool-result turn in a chain loop. - - Takes the prior response's full input chain + its structured - output, then appends a tool-role message carrying the new - ToolResult outputs. Attachments (e.g. images returned by tools) - are folded into a subsequent user-role message. - - This is what gives ``response.prompt.messages`` on the tool- - result turn the complete history for the next provider call — - including any reasoning signatures or thoughtSignatures from the - prior turn. - """ - from .parts import ( - AttachmentPart, - Message, - TextPart, - ToolResultPart, - ) - - chain: List[Any] = list(prior_response.prompt.messages) + list( - prior_response.messages - ) - if tool_results: - chain.append( - Message( - role="tool", - parts=[ - ToolResultPart( - name=tr.name, - output=tr.output, - tool_call_id=tr.tool_call_id, - ) - for tr in tool_results - ], - ) - ) - # Attachments that came back from tools ride on a trailing user - # message (mimics the legacy attachments=[] kwarg behavior). - if attachments: - chain.append( - Message( - role="user", - parts=[AttachmentPart(attachment=a) for a in attachments], - ) - ) - return chain - - -class _BaseChainResponse: - prompt: "Prompt" - stream: bool - conversation: Optional["_BaseConversation"] = None - _key: Optional[str] = None - - def __init__( - self, - prompt: Prompt, - model: "_BaseModel", - stream: bool, - conversation: _BaseConversation, - key: Optional[str] = None, - chain_limit: Optional[int] = 10, - before_call: Optional[Union[BeforeCallSync, BeforeCallAsync]] = None, - after_call: Optional[Union[AfterCallSync, AfterCallAsync]] = None, - ): - self.prompt = prompt - self.model = model - self.stream = stream - self._key = key - self._responses: List[Any] = [] - self.conversation = conversation - self.chain_limit = chain_limit - self.before_call = before_call - self.after_call = after_call - - def log_to_db(self, db): - for response in self._responses: - if isinstance(response, AsyncResponse): - sync_response = asyncio.run(response.to_sync_response()) - elif isinstance(response, Response): - sync_response = response - else: - assert False, "Should have been a Response or AsyncResponse" - sync_response.log_to_db(db) - - -class ChainResponse(_BaseChainResponse): - _responses: List["Response"] - before_call: Optional[BeforeCallSync] = None - after_call: Optional[AfterCallSync] = None - - def responses(self) -> Iterator[Response]: - prompt = self.prompt - count = 0 - current_response: Optional[Response] = Response( - prompt, - self.model, - self.stream, - key=self._key, - conversation=self.conversation, - ) - while current_response: - count += 1 - yield current_response - self._responses.append(current_response) - if self.chain_limit and count >= self.chain_limit: - raise ValueError(f"Chain limit of {self.chain_limit} exceeded.") - - # This could raise llm.CancelToolCall: - tool_results = current_response.execute_tool_calls( - before_call=self.before_call, after_call=self.after_call - ) - attachments = [] - for tool_result in tool_results: - attachments.extend(tool_result.attachments) - if tool_results: - # Pre-bake the full chain for the tool-result turn so - # response.prompt.messages is what gets sent — carries - # thoughtSignatures, thinking signatures, and everything - # else the model needs for the next call. - next_chain = _chain_for_tool_results( - current_response, tool_results, attachments - ) - current_response = Response( - Prompt( - "", # Next prompt text is empty; tool_results drive it - self.model, - tools=current_response.prompt.tools, - tool_results=tool_results, - messages=next_chain, - # Carry system + system_fragments forward so - # stateless-per-turn adapters (OpenAI and - # friends that read prompt.system directly) - # keep seeing the system prompt on every call - # of the chain loop. - system=self.prompt._system, - system_fragments=self.prompt.system_fragments, - options=self.prompt.options, - attachments=attachments, - ), - self.model, - stream=self.stream, - key=self._key, - conversation=self.conversation, - ) - else: - current_response = None - break - - def __iter__(self) -> Iterator[str]: - for response_item in self.responses(): - yield from response_item - - def stream_events(self): - "Yield StreamEvents from every response in the chain." - for response_item in self.responses(): - yield from response_item.stream_events() - - def text(self) -> str: - return "".join(self) - - -class AsyncChainResponse(_BaseChainResponse): - _responses: List["AsyncResponse"] - before_call: Optional[BeforeCallAsync] = None - after_call: Optional[AfterCallAsync] = None - - async def responses(self) -> AsyncIterator[AsyncResponse]: - prompt = self.prompt - count = 0 - current_response: Optional[AsyncResponse] = AsyncResponse( - prompt, - self.model, - self.stream, - key=self._key, - conversation=self.conversation, - ) - while current_response: - count += 1 - yield current_response - self._responses.append(current_response) - - if self.chain_limit and count >= self.chain_limit: - raise ValueError(f"Chain limit of {self.chain_limit} exceeded.") - - # This could raise llm.CancelToolCall: - tool_results = await current_response.execute_tool_calls( - before_call=self.before_call, after_call=self.after_call - ) - if tool_results: - attachments = [] - for tool_result in tool_results: - attachments.extend(tool_result.attachments) - # Pre-bake chain so prompt.messages carries full history - # + any thinking/tool-call signatures from prior turn. - next_chain = _chain_for_tool_results( - current_response, tool_results, attachments - ) - prompt = Prompt( - "", - self.model, - tools=current_response.prompt.tools, - tool_results=tool_results, - messages=next_chain, - # Carry system + system_fragments forward — same - # reasoning as the sync path. - system=self.prompt._system, - system_fragments=self.prompt.system_fragments, - options=self.prompt.options, - attachments=attachments, - ) - current_response = AsyncResponse( - prompt, - self.model, - stream=self.stream, - key=self._key, - conversation=self.conversation, - ) - else: - current_response = None - break - - async def __aiter__(self) -> AsyncIterator[str]: - async for response_item in self.responses(): - async for chunk in response_item: - yield chunk - - async def astream_events(self): - "Yield StreamEvents from every response in the chain." - async for response_item in self.responses(): - async for event in response_item.astream_events(): - yield event - - async def text(self) -> str: - all_chunks = [] - async for chunk in self: - all_chunks.append(chunk) - return "".join(all_chunks) - - -class Options(BaseModel): - model_config = ConfigDict(extra="forbid") - - -_Options = Options - - -class _get_key_mixin: - needs_key: Optional[str] = None - key: Optional[str] = None - key_env_var: Optional[str] = None - - def get_key(self, explicit_key: Optional[str] = None) -> Optional[str]: - from llm import get_key - - if self.needs_key is None: - # This model doesn't use an API key - return None - - if self.key is not None: - # Someone already set model.key='...' - return self.key - - # Attempt to load a key using llm.get_key() - key_value = get_key( - explicit_key=explicit_key, - key_alias=self.needs_key, - env_var=self.key_env_var, - ) - if key_value: - return key_value - - # Show a useful error message - message = "No key found - add one using 'llm keys set {}'".format( - self.needs_key - ) - if self.key_env_var: - message += " or set the {} environment variable".format(self.key_env_var) - raise NeedsKeyException(message) - - -class _BaseModel(ABC, _get_key_mixin): - model_id: str - can_stream: bool = False - attachment_types: Set = set() - - supports_schema = False - supports_tools = False - - class Options(_Options): - pass - - def _validate_attachments( - self, attachments: Optional[List[Attachment]] = None - ) -> None: - if attachments and not self.attachment_types: - raise ValueError("This model does not support attachments") - for attachment in attachments or []: - attachment_type = attachment.resolve_type() - if attachment_type not in self.attachment_types: - raise ValueError( - f"This model does not support attachments of type '{attachment_type}', " - f"only {', '.join(self.attachment_types)}" - ) - - def __str__(self) -> str: - return "{}{}: {}".format( - self.__class__.__name__, - " (async)" if isinstance(self, (AsyncModel, AsyncKeyModel)) else "", - self.model_id, - ) - - def __repr__(self) -> str: - return f"<{str(self)}>" - - -class _Model(_BaseModel): - def conversation( - self, - tools: Optional[List[ToolDef]] = None, - before_call: Optional[BeforeCallSync] = None, - after_call: Optional[AfterCallSync] = None, - chain_limit: Optional[int] = None, - ) -> Conversation: - return Conversation( - model=self, - tools=tools, - before_call=before_call, - after_call=after_call, - chain_limit=chain_limit, - ) - - def prompt( - self, - prompt: Optional[str] = None, - *, - fragments: Optional[List[Union[str, Fragment]]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - system_fragments: Optional[List[Union[str, Fragment]]] = None, - messages: Optional[List[Any]] = None, - stream: bool = True, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - **options, - ) -> Response: - key_value = options.pop("key", None) - self._validate_attachments(attachments) - return Response( - Prompt( - prompt, - fragments=fragments, - attachments=attachments, - system=system, - schema=schema, - tools=tools, - tool_results=tool_results, - system_fragments=system_fragments, - messages=messages, - model=self, - options=self.Options(**options), - ), - self, - stream, - key=key_value, - ) - - def chain( - self, - prompt: Optional[str] = None, - *, - fragments: Optional[List[str]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - system_fragments: Optional[List[str]] = None, - messages: Optional[List[Any]] = None, - stream: bool = True, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - before_call: Optional[BeforeCallSync] = None, - after_call: Optional[AfterCallSync] = None, - key: Optional[str] = None, - options: Optional[dict] = None, - ) -> ChainResponse: - return self.conversation().chain( - prompt=prompt, - fragments=fragments, - attachments=attachments, - system=system, - system_fragments=system_fragments, - messages=messages, - stream=stream, - schema=schema, - tools=tools, - tool_results=tool_results, - before_call=before_call, - after_call=after_call, - key=key, - options=options, - ) - - -class Model(_Model): - @abstractmethod - def execute( - self, - prompt: Prompt, - stream: bool, - response: Response, - conversation: Optional[Conversation], - ) -> Iterator[str]: - pass - - -class KeyModel(_Model): - @abstractmethod - def execute( - self, - prompt: Prompt, - stream: bool, - response: Response, - conversation: Optional[Conversation], - key: Optional[str], - ) -> Iterator[str]: - pass - - -class _AsyncModel(_BaseModel): - def conversation( - self, - tools: Optional[List[ToolDef]] = None, - before_call: Optional[BeforeCallAsync] = None, - after_call: Optional[AfterCallAsync] = None, - chain_limit: Optional[int] = None, - ) -> AsyncConversation: - return AsyncConversation( - model=self, - tools=tools, - before_call=before_call, - after_call=after_call, - chain_limit=chain_limit, - ) - - def prompt( - self, - prompt: Optional[str] = None, - *, - fragments: Optional[List[Union[str, Fragment]]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - system_fragments: Optional[List[Union[str, Fragment]]] = None, - messages: Optional[List[Any]] = None, - stream: bool = True, - **options, - ) -> AsyncResponse: - key_value = options.pop("key", None) - self._validate_attachments(attachments) - return AsyncResponse( - Prompt( - prompt, - fragments=fragments, - attachments=attachments, - system=system, - schema=schema, - tools=tools, - tool_results=tool_results, - system_fragments=system_fragments, - messages=messages, - model=self, - options=self.Options(**options), - ), - self, - stream, - key=key_value, - ) - - def chain( - self, - prompt: Optional[str] = None, - *, - fragments: Optional[List[str]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - system_fragments: Optional[List[str]] = None, - messages: Optional[List[Any]] = None, - stream: bool = True, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - before_call: Optional[BeforeCallAsync] = None, - after_call: Optional[AfterCallAsync] = None, - key: Optional[str] = None, - options: Optional[dict] = None, - ) -> AsyncChainResponse: - return self.conversation().chain( - prompt=prompt, - fragments=fragments, - attachments=attachments, - system=system, - system_fragments=system_fragments, - messages=messages, - stream=stream, - schema=schema, - tools=tools, - tool_results=tool_results, - before_call=before_call, - after_call=after_call, - key=key, - options=options, - ) - - -class AsyncModel(_AsyncModel): - @abstractmethod - async def execute( - self, - prompt: Prompt, - stream: bool, - response: AsyncResponse, - conversation: Optional[AsyncConversation], - ) -> AsyncGenerator[str, None]: - if False: # Ensure it's a generator type - yield "" - pass - - -class AsyncKeyModel(_AsyncModel): - @abstractmethod - async def execute( - self, - prompt: Prompt, - stream: bool, - response: AsyncResponse, - conversation: Optional[AsyncConversation], - key: Optional[str], - ) -> AsyncGenerator[str, None]: - if False: # Ensure it's a generator type - yield "" - pass - - -class EmbeddingModel(ABC, _get_key_mixin): - model_id: str - key: Optional[str] = None - needs_key: Optional[str] = None - key_env_var: Optional[str] = None - supports_text: bool = True - supports_binary: bool = False - batch_size: Optional[int] = None - - def _check(self, item: Union[str, bytes]): - if not self.supports_binary and isinstance(item, bytes): - raise ValueError( - "This model does not support binary data, only text strings" - ) - if not self.supports_text and isinstance(item, str): - raise ValueError( - "This model does not support text strings, only binary data" - ) - - def embed(self, item: Union[str, bytes]) -> List[float]: - "Embed a single text string or binary blob, return a list of floats" - self._check(item) - return next(iter(self.embed_batch([item]))) - - def embed_multi( - self, items: Iterable[Union[str, bytes]], batch_size: Optional[int] = None - ) -> Iterator[List[float]]: - "Embed multiple items in batches according to the model batch_size" - iter_items = iter(items) - effective_batch_size = self.batch_size if batch_size is None else batch_size - if (not self.supports_binary) or (not self.supports_text): - - def checking_iter(inner_items): - for item_to_check in inner_items: - self._check(item_to_check) - yield item_to_check - - iter_items = checking_iter(items) - if effective_batch_size is None: - yield from self.embed_batch(iter_items) - return - while True: - batch_items = list(islice(iter_items, effective_batch_size)) - if not batch_items: - break - yield from self.embed_batch(batch_items) - - @abstractmethod - def embed_batch(self, items: Iterable[Union[str, bytes]]) -> Iterator[List[float]]: - """ - Embed a batch of strings or blobs, return a list of lists of floats - """ - pass - - def __str__(self) -> str: - return "{}: {}".format(self.__class__.__name__, self.model_id) - - def __repr__(self) -> str: - return f"<{str(self)}>" - - -@dataclass -class ModelWithAliases: - "A model with its optional async counterpart and aliases." - - model: Model - async_model: AsyncModel - aliases: Set[str] - - def matches(self, query: str) -> bool: - query_lower = query.lower() - all_strings: List[str] = [] - all_strings.extend(self.aliases) - if self.model: - all_strings.append(str(self.model)) - if self.async_model: - all_strings.append(str(self.async_model.model_id)) - return any(query_lower in alias.lower() for alias in all_strings) - - -@dataclass -class EmbeddingModelWithAliases: - model: EmbeddingModel - aliases: Set[str] - - def matches(self, query: str) -> bool: - query_lower = query.lower() - all_strings: List[str] = [] - all_strings.extend(self.aliases) - all_strings.append(str(self.model)) - return any(query_lower in alias.lower() for alias in all_strings) - - -def _conversation_name(text): - # Collapse whitespace, including newlines - text = re.sub(r"\s+", " ", text) - if len(text) <= CONVERSATION_NAME_LENGTH: - return text - return text[: CONVERSATION_NAME_LENGTH - 1] + "…" - - -def _ensure_dict_schema(schema): - """Convert a Pydantic model to a JSON schema dict if needed.""" - if schema and not isinstance(schema, dict) and issubclass(schema, BaseModel): - schema_dict = schema.model_json_schema() - _remove_titles_recursively(schema_dict) - return schema_dict - return schema - - -def _remove_titles_recursively(obj): - """Recursively remove all 'title' fields from a nested dictionary.""" - if isinstance(obj, dict): - # Remove title if present - obj.pop("title", None) - - # Recursively process all values - for value in obj.values(): - _remove_titles_recursively(value) - elif isinstance(obj, list): - # Process each item in lists - for item in obj: - _remove_titles_recursively(item) - - -def _get_instance(implementation): - if hasattr(implementation, "__self__"): - return implementation.__self__ - return None diff --git a/build/lib/llm/parts.py b/build/lib/llm/parts.py deleted file mode 100644 index e627f876b..000000000 --- a/build/lib/llm/parts.py +++ /dev/null @@ -1,340 +0,0 @@ -"""Part, Message, and StreamEvent value types. - -Parts represent the structured content of model interactions: text, -reasoning, tool calls, tool results, and attachments. A Message wraps a -list of Parts with a role. StreamEvent wraps a streaming chunk with type -information so consumers can distinguish text from reasoning from tool -call fragments as they arrive. - -These types are pure values — identity (ids, parent links, storage keys) -is a storage concern that lives elsewhere. Two Messages with identical -content are equal. -""" - -import base64 -from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional - -from .models import Attachment -from .serialization import ( - AttachmentDict, - AttachmentPartDict, - MessageDict, - PartDict, - ReasoningPartDict, - TextPartDict, - ToolCallPartDict, - ToolResultPartDict, -) - - -def _attachment_to_dict(att: Attachment) -> AttachmentDict: - d: Dict[str, Any] = {} - if att.type: - d["type"] = att.type - if att.url: - d["url"] = att.url - if att.path: - d["path"] = att.path - if att.content: - d["content"] = base64.b64encode(att.content).decode("ascii") - return d # type: ignore[return-value] - - -def _attachment_from_dict(d: AttachmentDict) -> Attachment: - content = d.get("content") - if isinstance(content, str): - content = base64.b64decode(content) - return Attachment( - type=d.get("type"), - path=d.get("path"), - url=d.get("url"), - content=content, - ) - - -@dataclass -class Part: - """Base class for all parts. Role lives on the enclosing Message.""" - - def to_dict(self) -> PartDict: - raise NotImplementedError - - @staticmethod - def from_dict(d: PartDict) -> "Part": - type_ = d.get("type") - pm = d.get("provider_metadata") - if type_ == "text": - return TextPart(text=d.get("text", ""), provider_metadata=pm) - if type_ == "reasoning": - return ReasoningPart( - text=d.get("text", ""), - redacted=d.get("redacted", False), - token_count=d.get("token_count"), - provider_metadata=pm, - ) - if type_ == "tool_call": - return ToolCallPart( - name=d["name"], - arguments=d.get("arguments", {}), - tool_call_id=d.get("tool_call_id"), - server_executed=d.get("server_executed", False), - provider_metadata=pm, - ) - if type_ == "tool_result": - return ToolResultPart( - name=d["name"], - output=d.get("output", ""), - tool_call_id=d.get("tool_call_id"), - server_executed=d.get("server_executed", False), - exception=d.get("exception"), - attachments=[ - _attachment_from_dict(a) for a in d.get("attachments", []) - ], - provider_metadata=pm, - ) - if type_ == "attachment": - att_dict = d.get("attachment") - attachment = _attachment_from_dict(att_dict) if att_dict else None - return AttachmentPart(attachment=attachment, provider_metadata=pm) - raise ValueError(f"Unknown part type: {type_!r}") - - -@dataclass -class TextPart(Part): - text: str = "" - provider_metadata: Optional[Dict[str, Any]] = None - - def to_dict(self) -> TextPartDict: - d: Dict[str, Any] = {"type": "text", "text": self.text} - if self.provider_metadata: - d["provider_metadata"] = self.provider_metadata - return d # type: ignore[return-value] - - -@dataclass -class ReasoningPart(Part): - """Reasoning/thinking tokens from the model. - - `redacted=True, text=""` represents the opaque-token-count case - (OpenAI GPT-5 series, Gemini) where the provider reports only a - count, not content. - """ - - text: str = "" - redacted: bool = False - token_count: Optional[int] = None - provider_metadata: Optional[Dict[str, Any]] = None - - def to_dict(self) -> ReasoningPartDict: - d: Dict[str, Any] = {"type": "reasoning", "text": self.text} - if self.redacted: - d["redacted"] = True - if self.token_count is not None: - d["token_count"] = self.token_count - if self.provider_metadata: - d["provider_metadata"] = self.provider_metadata - return d # type: ignore[return-value] - - -@dataclass -class ToolCallPart(Part): - """A request by the model to call a tool. - - `server_executed=True` marks calls the provider executed on the - server (Anthropic web search, Gemini code execution) rather than - the LLM tool framework. - """ - - name: str = "" - arguments: Dict[str, Any] = field(default_factory=dict) - tool_call_id: Optional[str] = None - server_executed: bool = False - provider_metadata: Optional[Dict[str, Any]] = None - - def to_dict(self) -> ToolCallPartDict: - d: Dict[str, Any] = { - "type": "tool_call", - "name": self.name, - "arguments": self.arguments, - } - if self.tool_call_id is not None: - d["tool_call_id"] = self.tool_call_id - if self.server_executed: - d["server_executed"] = True - if self.provider_metadata: - d["provider_metadata"] = self.provider_metadata - return d # type: ignore[return-value] - - -@dataclass -class ToolResultPart(Part): - """The result of a tool call.""" - - name: str = "" - output: str = "" - tool_call_id: Optional[str] = None - server_executed: bool = False - attachments: List[Any] = field(default_factory=list) - exception: Optional[str] = None - provider_metadata: Optional[Dict[str, Any]] = None - - def to_dict(self) -> ToolResultPartDict: - d: Dict[str, Any] = { - "type": "tool_result", - "name": self.name, - "output": self.output, - } - if self.tool_call_id is not None: - d["tool_call_id"] = self.tool_call_id - if self.server_executed: - d["server_executed"] = True - if self.exception is not None: - d["exception"] = self.exception - if self.attachments: - d["attachments"] = [_attachment_to_dict(a) for a in self.attachments] - if self.provider_metadata: - d["provider_metadata"] = self.provider_metadata - return d # type: ignore[return-value] - - -@dataclass -class AttachmentPart(Part): - """An inline attachment (image, audio, file).""" - - attachment: Optional[Attachment] = None - provider_metadata: Optional[Dict[str, Any]] = None - - def to_dict(self) -> AttachmentPartDict: - d: Dict[str, Any] = {"type": "attachment"} - if self.attachment: - d["attachment"] = _attachment_to_dict(self.attachment) - if self.provider_metadata: - d["provider_metadata"] = self.provider_metadata - return d # type: ignore[return-value] - - -@dataclass -class Message: - """A single turn in a conversation: role + list of parts. - - `parts` contains one or more Part objects. `provider_metadata` - carries opaque provider-specific data attached to the message as a - whole; part-level data lives on the individual Part's - `provider_metadata`. - """ - - role: str - parts: List[Part] = field(default_factory=list) - provider_metadata: Optional[Dict[str, Any]] = None - - def to_dict(self) -> MessageDict: - d: Dict[str, Any] = { - "role": self.role, - "parts": [p.to_dict() for p in self.parts], - } - if self.provider_metadata: - d["provider_metadata"] = self.provider_metadata - return d # type: ignore[return-value] - - @staticmethod - def from_dict(d: MessageDict) -> "Message": - return Message( - role=d["role"], - parts=[Part.from_dict(p) for p in d.get("parts", [])], - provider_metadata=d.get("provider_metadata"), - ) - - -def normalize_parts(items: Any) -> List[Part]: - """Normalize helper inputs to a list of Part objects. - - Accepts str (→ TextPart), Attachment (→ AttachmentPart), Part - (passed through), or a list/tuple of those (flattened one level). - """ - out: List[Part] = [] - for item in items: - if isinstance(item, Part): - out.append(item) - elif isinstance(item, str): - out.append(TextPart(text=item)) - elif isinstance(item, Attachment): - out.append(AttachmentPart(attachment=item)) - elif isinstance(item, (list, tuple)): - out.extend(normalize_parts(item)) - else: - raise TypeError(f"Cannot convert {item!r} to an llm Part") - return out - - -def system( - *items: Any, provider_metadata: Optional[Dict[str, Any]] = None -) -> Message: - "Build a Message with role='system'." - return Message( - role="system", - parts=normalize_parts(items), - provider_metadata=provider_metadata, - ) - - -def user( - *items: Any, provider_metadata: Optional[Dict[str, Any]] = None -) -> Message: - "Build a Message with role='user'." - return Message( - role="user", - parts=normalize_parts(items), - provider_metadata=provider_metadata, - ) - - -def assistant( - *items: Any, provider_metadata: Optional[Dict[str, Any]] = None -) -> Message: - "Build a Message with role='assistant'." - return Message( - role="assistant", - parts=normalize_parts(items), - provider_metadata=provider_metadata, - ) - - -def tool_message( - *items: Any, provider_metadata: Optional[Dict[str, Any]] = None -) -> Message: - "Build a Message with role='tool' (typically wrapping ToolResultParts)." - return Message( - role="tool", - parts=normalize_parts(items), - provider_metadata=provider_metadata, - ) - - -@dataclass -class StreamEvent: - """A streaming event from a model response. - - `part_index` groups events into parts — events sharing an index - belong to the same logical part. Mixing families (e.g. text with - tool_call_name) at the same index is a plugin bug. - - `provider_metadata` carries opaque provider data (Anthropic - `signature`, Gemini `thoughtSignature`, OpenAI `encrypted_content`) - that must be echoed back on the next request; the framework merges - it onto the finalized Part (last non-None wins per top-level key). - - `message_index` is for providers that emit multiple assistant - messages in a single response (Anthropic server-side tool - execution); most plugins leave it at 0. - """ - - type: str # "text" / "reasoning" / "tool_call_name" / - # "tool_call_args" / "tool_result" - chunk: str - part_index: int - tool_call_id: Optional[str] = None - server_executed: bool = False - tool_name: Optional[str] = None - provider_metadata: Optional[Dict[str, Any]] = None - message_index: int = 0 diff --git a/build/lib/llm/plugins.py b/build/lib/llm/plugins.py deleted file mode 100644 index 0125ede04..000000000 --- a/build/lib/llm/plugins.py +++ /dev/null @@ -1,50 +0,0 @@ -import importlib -from importlib import metadata -import os -import pluggy -import sys -from . import hookspecs - -DEFAULT_PLUGINS = ( - "llm.default_plugins.openai_models", - "llm.default_plugins.default_tools", -) - -pm = pluggy.PluginManager("llm") -pm.add_hookspecs(hookspecs) - -LLM_LOAD_PLUGINS = os.environ.get("LLM_LOAD_PLUGINS", None) - -_loaded = False - - -def load_plugins(): - global _loaded - if _loaded: - return - _loaded = True - if not hasattr(sys, "_called_from_test") and LLM_LOAD_PLUGINS is None: - # Only load plugins if not running tests - pm.load_setuptools_entrypoints("llm") - - # Load any plugins specified in LLM_LOAD_PLUGINS") - if LLM_LOAD_PLUGINS is not None: - for package_name in [ - name for name in LLM_LOAD_PLUGINS.split(",") if name.strip() - ]: - try: - distribution = metadata.distribution(package_name) # Updated call - llm_entry_points = [ - ep for ep in distribution.entry_points if ep.group == "llm" - ] - for entry_point in llm_entry_points: - mod = entry_point.load() - pm.register(mod, name=entry_point.name) - # Ensure name can be found in plugin_to_distinfo later: - pm._plugin_distinfo.append((mod, distribution)) # type: ignore - except metadata.PackageNotFoundError: - sys.stderr.write(f"Plugin {package_name} could not be found\n") - - for plugin in DEFAULT_PLUGINS: - mod = importlib.import_module(plugin) - pm.register(mod, plugin) diff --git a/build/lib/llm/py.typed b/build/lib/llm/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/build/lib/llm/serialization.py b/build/lib/llm/serialization.py deleted file mode 100644 index b6b4bf972..000000000 --- a/build/lib/llm/serialization.py +++ /dev/null @@ -1,182 +0,0 @@ -"""TypedDict spec for the JSON-safe wire form of Part, Message, and Response. - -These are the exact shapes returned by ``Part.to_dict()``, -``Message.to_dict()``, and ``Response.to_dict()`` — and accepted by the -matching ``from_dict`` classmethods. They are the canonical wire format; -use them to annotate any code that reads or writes serialized llm data. - -Example:: - - from llm.serialization import MessageDict - - def save_messages(conn, messages: list[MessageDict]) -> None: - for m in messages: - conn.execute( - "INSERT INTO messages(role, parts_json) VALUES (?, ?)", - (m["role"], json.dumps(m["parts"])), - ) - -Or pair with Pydantic's TypeAdapter for runtime validation:: - - from pydantic import TypeAdapter - from llm.serialization import MessageDict - - msg = TypeAdapter(MessageDict).validate_python(incoming_dict) - -Or export JSON Schema for cross-language consumers:: - - schema = TypeAdapter(MessageDict).json_schema() - -The TypedDicts are erased at runtime — zero overhead. ``NotRequired`` -keys may be absent from a serialized payload; required keys must -always be present. -""" - -from typing import Any, Dict, List, Literal, Union - -# NotRequired moved to typing in 3.11; use typing_extensions for 3.10 -# support. typing_extensions is a transitive dep via pydantic. -from typing_extensions import NotRequired, TypedDict - - -__all__ = [ - "AttachmentDict", - "AttachmentPartDict", - "MessageDict", - "PartDict", - "PromptDict", - "ReasoningPartDict", - "ResponseDict", - "TextPartDict", - "ToolCallPartDict", - "ToolResultPartDict", - "UsageDict", -] - - -# ---- Attachment payload (nested inside AttachmentPartDict + tool results) ---- - - -class AttachmentDict(TypedDict, total=False): - """Nested attachment payload. All fields optional — an Attachment - may carry a type, a url, a path, and/or base64-encoded content. - """ - - type: str - url: str - path: str - # base64-encoded bytes when the attachment was constructed with raw - # content= bytes. - content: str - - -# ---- Per-Part TypedDicts (discriminated by the `type` field) ----------------- - - -class TextPartDict(TypedDict): - type: Literal["text"] - text: str - provider_metadata: NotRequired[Dict[str, Any]] - - -class ReasoningPartDict(TypedDict): - type: Literal["reasoning"] - text: str - # Redacted reasoning: text is "" and token_count carries the opaque - # count reported by the provider (OpenAI GPT-5, Gemini thinking). - redacted: NotRequired[bool] - token_count: NotRequired[int] - provider_metadata: NotRequired[Dict[str, Any]] - - -class ToolCallPartDict(TypedDict): - type: Literal["tool_call"] - name: str - arguments: Dict[str, Any] - tool_call_id: NotRequired[str] - # True for provider-executed calls (Anthropic web search, Gemini code - # execution). Client echoes the block back as-is on next turn. - server_executed: NotRequired[bool] - provider_metadata: NotRequired[Dict[str, Any]] - - -class ToolResultPartDict(TypedDict): - type: Literal["tool_result"] - name: str - output: str - tool_call_id: NotRequired[str] - server_executed: NotRequired[bool] - exception: NotRequired[str] - attachments: NotRequired[List[AttachmentDict]] - provider_metadata: NotRequired[Dict[str, Any]] - - -class AttachmentPartDict(TypedDict): - type: Literal["attachment"] - attachment: NotRequired[AttachmentDict] - provider_metadata: NotRequired[Dict[str, Any]] - - -PartDict = Union[ - TextPartDict, - ReasoningPartDict, - ToolCallPartDict, - ToolResultPartDict, - AttachmentPartDict, -] -"""Discriminated union of Part dict shapes. Use with -``pydantic.TypeAdapter(PartDict)`` to validate / dispatch by ``type``. -""" - - -# ---- Message ---------------------------------------------------------------- - - -class MessageDict(TypedDict): - """JSON-safe form of ``llm.Message``. - - ``role`` is one of "user", "assistant", "system", "tool" in practice - — typed as ``str`` here to leave room for provider-specific values. - """ - - role: str - parts: List[PartDict] - provider_metadata: NotRequired[Dict[str, Any]] - - -# ---- Response + nested shapes ----------------------------------------------- - - -class PromptDict(TypedDict): - """The ``prompt`` sub-dict of ``Response.to_dict()`` — captures the - full input chain that was sent for this turn plus any options that - apply.""" - - messages: List[MessageDict] - options: NotRequired[Dict[str, Any]] - system: NotRequired[str] - - -class UsageDict(TypedDict, total=False): - """Optional usage block on ``ResponseDict``. All fields optional; - providers vary in which they report.""" - - input: int - output: int - details: Dict[str, Any] - - -class ResponseDict(TypedDict): - """JSON-safe form of ``llm.Response`` — everything needed for - ``Response.from_dict`` to rehydrate and ``response.reply()`` to - continue a conversation across a process boundary. - """ - - model: str - prompt: PromptDict - messages: List[MessageDict] - # Audit fields — present on a freshly-serialized response, optional - # on hand-constructed ones. - id: NotRequired[str] - usage: NotRequired[UsageDict] - datetime_utc: NotRequired[str] diff --git a/build/lib/llm/templates.py b/build/lib/llm/templates.py deleted file mode 100644 index ac1b7c716..000000000 --- a/build/lib/llm/templates.py +++ /dev/null @@ -1,92 +0,0 @@ -from pydantic import BaseModel, ConfigDict -import string -from typing import Optional, Any, Dict, List, Tuple - - -class AttachmentType(BaseModel): - type: str - value: str - - -class Template(BaseModel): - """A reusable prompt template.""" - - name: str - prompt: Optional[str] = None - system: Optional[str] = None - attachments: Optional[List[str]] = None - attachment_types: Optional[List[AttachmentType]] = None - model: Optional[str] = None - defaults: Optional[Dict[str, Any]] = None - options: Optional[Dict[str, Any]] = None - extract: Optional[bool] = None # For extracting fenced code blocks - extract_last: Optional[bool] = None - schema_object: Optional[dict] = None - fragments: Optional[List[str]] = None - system_fragments: Optional[List[str]] = None - tools: Optional[List[str]] = None - functions: Optional[str] = None - - model_config = ConfigDict(extra="forbid") - - class MissingVariables(Exception): - pass - - def __init__(self, **data): - super().__init__(**data) - # Not a pydantic field to avoid YAML being able to set it - # this controls if Python inline functions code is trusted - self._functions_is_trusted = False - - def evaluate( - self, input: str, params: Optional[Dict[str, Any]] = None - ) -> Tuple[Optional[str], Optional[str]]: - """Evaluate the template with the given input and parameters, returning (prompt, system).""" - params = params or {} - params["input"] = input - if self.defaults: - for k, v in self.defaults.items(): - if k not in params: - params[k] = v - prompt: Optional[str] = None - system: Optional[str] = None - if not self.prompt: - system = self.interpolate(self.system, params) - prompt = input - else: - prompt = self.interpolate(self.prompt, params) - system = self.interpolate(self.system, params) - return prompt, system - - def vars(self) -> set: - """Return the set of variable names used in the prompt and system templates.""" - all_vars = set() - for text in [self.prompt, self.system]: - if not text: - continue - all_vars.update(self.extract_vars(string.Template(text))) - return all_vars - - @classmethod - def interpolate(cls, text: Optional[str], params: Dict[str, Any]) -> Optional[str]: - """Substitute template variables in text with values from params, raising MissingVariables if any are absent.""" - if not text: - return text - # Confirm all variables in text are provided - string_template = string.Template(text) - vars = cls.extract_vars(string_template) - missing = [p for p in vars if p not in params] - if missing: - raise cls.MissingVariables( - "Missing variables: {}".format(", ".join(missing)) - ) - return string_template.substitute(**params) - - @staticmethod - def extract_vars(string_template: string.Template) -> List[str]: - """Extract and return the list of named variable identifiers from a string.Template.""" - return [ - match.group("named") - for match in string_template.pattern.finditer(string_template.template) - if match.group("named") - ] diff --git a/build/lib/llm/tools.py b/build/lib/llm/tools.py deleted file mode 100644 index 5ac0a7dcb..000000000 --- a/build/lib/llm/tools.py +++ /dev/null @@ -1,37 +0,0 @@ -from datetime import datetime, timezone -from importlib.metadata import version -import time - - -def llm_version() -> str: - "Return the installed version of llm" - return version("llm") - - -def llm_time() -> dict: - "Returns the current time, as local time and UTC" - # Get current times - utc_time = datetime.now(timezone.utc) - local_time = datetime.now() - - # Get timezone information - local_tz_name = time.tzname[time.localtime().tm_isdst] - is_dst = bool(time.localtime().tm_isdst) - - # Calculate offset - offset_seconds = -time.timezone if not is_dst else -time.altzone - offset_hours = offset_seconds // 3600 - offset_minutes = (offset_seconds % 3600) // 60 - - timezone_offset = ( - f"UTC{'+' if offset_hours >= 0 else ''}{offset_hours:02d}:{offset_minutes:02d}" - ) - - return { - "utc_time": utc_time.strftime("%Y-%m-%d %H:%M:%S UTC"), - "utc_time_iso": utc_time.isoformat(), - "local_timezone": local_tz_name, - "local_time": local_time.strftime("%Y-%m-%d %H:%M:%S"), - "timezone_offset": timezone_offset, - "is_dst": is_dst, - } diff --git a/build/lib/llm/utils.py b/build/lib/llm/utils.py deleted file mode 100644 index 587f19284..000000000 --- a/build/lib/llm/utils.py +++ /dev/null @@ -1,735 +0,0 @@ -import click -import hashlib -import httpx -import itertools -import json -import pathlib -import puremagic -import re -import sqlite_utils -import textwrap -from typing import Any, List, Dict, Optional, Tuple, Type -import os -import threading -import time -from typing import Final - -from ulid import ULID - -MIME_TYPE_FIXES = { - "audio/wave": "audio/wav", -} - - -class Fragment(str): - def __new__(cls, content, *args, **kwargs): - # For immutable classes like str, __new__ creates the string object - return super().__new__(cls, content) - - def __init__(self, content, source=""): - # Initialize our custom attributes - self.source = source - - def id(self): - return hashlib.sha256(self.encode("utf-8")).hexdigest() - - -def mimetype_from_string(content) -> Optional[str]: - try: - type_ = puremagic.from_string(content, mime=True) - return MIME_TYPE_FIXES.get(type_, type_) - except puremagic.PureError: - return None - - -def mimetype_from_path(path) -> Optional[str]: - try: - type_ = puremagic.from_file(path, mime=True) - return MIME_TYPE_FIXES.get(type_, type_) - except puremagic.PureError: - return None - - -def dicts_to_table_string( - headings: List[str], dicts: List[Dict[str, str]] -) -> List[str]: - max_lengths = [len(h) for h in headings] - - # Compute maximum length for each column - for d in dicts: - for i, h in enumerate(headings): - if h in d and len(str(d[h])) > max_lengths[i]: - max_lengths[i] = len(str(d[h])) - - # Generate formatted table strings - res = [] - res.append(" ".join(h.ljust(max_lengths[i]) for i, h in enumerate(headings))) - - for d in dicts: - row = [] - for i, h in enumerate(headings): - row.append(str(d.get(h, "")).ljust(max_lengths[i])) - res.append(" ".join(row)) - - return res - - -def remove_dict_none_values(d): - """ - Recursively remove keys with value of None or value of a dict that is all values of None - """ - if not isinstance(d, dict): - return d - new_dict = {} - for key, value in d.items(): - if value is not None: - if isinstance(value, dict): - nested = remove_dict_none_values(value) - if nested: - new_dict[key] = nested - elif isinstance(value, list): - new_dict[key] = [remove_dict_none_values(v) for v in value] - else: - new_dict[key] = value - return new_dict - - -class _LogResponse(httpx.Response): - def iter_bytes(self, *args, **kwargs): - for chunk in super().iter_bytes(*args, **kwargs): - click.echo(chunk.decode(), err=True) - yield chunk - - -class _LogTransport(httpx.BaseTransport): - def __init__(self, transport: httpx.BaseTransport): - self.transport = transport - - def handle_request(self, request: httpx.Request) -> httpx.Response: - response = self.transport.handle_request(request) - return _LogResponse( - status_code=response.status_code, - headers=response.headers, - stream=response.stream, - extensions=response.extensions, - ) - - -def _no_accept_encoding(request: httpx.Request): - request.headers.pop("accept-encoding", None) - - -def _log_response(response: httpx.Response): - request = response.request - click.echo(f"Request: {request.method} {request.url}", err=True) - click.echo(" Headers:", err=True) - for key, value in request.headers.items(): - if key.lower() == "authorization": - value = "[...]" - if key.lower() == "cookie": - value = value.split("=")[0] + "=..." - click.echo(f" {key}: {value}", err=True) - click.echo(" Body:", err=True) - try: - request_body = json.loads(request.content) - click.echo( - textwrap.indent(json.dumps(request_body, indent=2), " "), err=True - ) - except json.JSONDecodeError: - click.echo(textwrap.indent(request.content.decode(), " "), err=True) - click.echo(f"Response: status_code={response.status_code}", err=True) - click.echo(" Headers:", err=True) - for key, value in response.headers.items(): - if key.lower() == "set-cookie": - value = value.split("=")[0] + "=..." - click.echo(f" {key}: {value}", err=True) - click.echo(" Body:", err=True) - - -def logging_client() -> httpx.Client: - return httpx.Client( - transport=_LogTransport(httpx.HTTPTransport()), - event_hooks={"request": [_no_accept_encoding], "response": [_log_response]}, - ) - - -def simplify_usage_dict(d): - # Recursively remove keys with value 0 and empty dictionaries - def remove_empty_and_zero(obj): - if isinstance(obj, dict): - cleaned = { - k: remove_empty_and_zero(v) - for k, v in obj.items() - if v != 0 and v != {} - } - return {k: v for k, v in cleaned.items() if v is not None and v != {}} - return obj - - return remove_empty_and_zero(d) or {} - - -def token_usage_string(input_tokens, output_tokens, token_details) -> str: - bits = [] - if input_tokens is not None: - bits.append(f"{format(input_tokens, ',')} input") - if output_tokens is not None: - bits.append(f"{format(output_tokens, ',')} output") - if token_details: - bits.append(json.dumps(token_details)) - return ", ".join(bits) - - -def extract_fenced_code_block(text: str, last: bool = False) -> Optional[str]: - """ - Extracts and returns Markdown fenced code block found in the given text. - - The function handles fenced code blocks that: - - Use at least three backticks (`). - - May include a language tag immediately after the opening backticks. - - Use more than three backticks as long as the closing fence has the same number. - - If no fenced code block is found, the function returns None. - - Args: - text (str): The input text to search for a fenced code block. - last (bool): Extract the last code block if True, otherwise the first. - - Returns: - Optional[str]: The content of the fenced code block, or None if not found. - """ - # Regex pattern to match fenced code blocks - # - ^ or \n ensures that the fence is at the start of a line - # - (`{3,}) captures the opening backticks (at least three) - # - (\w+)? optionally captures the language tag - # - \n matches the newline after the opening fence - # - (.*?) non-greedy match for the code block content - # - (?P=fence) ensures that the closing fence has the same number of backticks - # - [ ]* allows for optional spaces between the closing fence and newline - # - (?=\n|$) ensures that the closing fence is followed by a newline or end of string - pattern = re.compile( - r"""(?m)^(?P`{3,})(?P\w+)?\n(?P.*?)^(?P=fence)[ ]*(?=\n|$)""", - re.DOTALL, - ) - matches = list(pattern.finditer(text)) - if matches: - match = matches[-1] if last else matches[0] - return match.group("code") - return None - - -def make_schema_id(schema: dict) -> Tuple[str, str]: - schema_json = json.dumps(schema, separators=(",", ":")) - schema_id = hashlib.blake2b(schema_json.encode(), digest_size=16).hexdigest() - return schema_id, schema_json - - -def output_rows_as_json(rows, nl=False, compact=False, json_cols=()): - """ - Output rows as JSON - either newline-delimited or an array - - Parameters: - - rows: Iterable of dictionaries to output - - nl: Boolean, if True, use newline-delimited JSON - - compact: Boolean, if True uses [{"...": "..."}\n {"...": "..."}] format - - json_cols: Iterable of columns that contain JSON - - Yields: - - Stream of strings to be output - """ - current_iter, next_iter = itertools.tee(rows, 2) - next(next_iter, None) - first = True - - for row, next_row in itertools.zip_longest(current_iter, next_iter): - is_last = next_row is None - for col in json_cols: - row[col] = json.loads(row[col]) - - if nl: - # Newline-delimited JSON: one JSON object per line - yield json.dumps(row) - elif compact: - # Compact array format: [{"...": "..."}\n {"...": "..."}] - yield "{firstchar}{serialized}{maybecomma}{lastchar}".format( - firstchar="[" if first else " ", - serialized=json.dumps(row), - maybecomma="," if not is_last else "", - lastchar="]" if is_last else "", - ) - else: - # Pretty-printed array format with indentation - yield "{firstchar}{serialized}{maybecomma}{lastchar}".format( - firstchar="[\n" if first else "", - serialized=textwrap.indent(json.dumps(row, indent=2), " "), - maybecomma="," if not is_last else "", - lastchar="\n]" if is_last else "", - ) - first = False - - if first and not nl: - # We didn't output any rows, so yield the empty list - yield "[]" - - -def resolve_schema_input(db, schema_input, load_template): - # schema_input might be JSON or a filepath or an ID or t:name - if not schema_input: - return - if schema_input.strip().startswith("t:"): - name = schema_input.strip()[2:] - schema_object = None - try: - template = load_template(name) - schema_object = template.schema_object - except ValueError: - raise click.ClickException("Invalid template: {}".format(name)) - if not schema_object: - raise click.ClickException("Template '{}' has no schema".format(name)) - return template.schema_object - if schema_input.strip().startswith("{"): - try: - return json.loads(schema_input) - except ValueError: - pass - if " " in schema_input.strip() or "," in schema_input: - # Treat it as schema DSL - return schema_dsl(schema_input) - # Is it a file on disk? - path = pathlib.Path(schema_input) - if path.exists(): - try: - return json.loads(path.read_text()) - except ValueError: - raise click.ClickException("Schema file contained invalid JSON") - # Last attempt: is it an ID in the DB? - try: - row = db["schemas"].get(schema_input) - return json.loads(row["content"]) - except (sqlite_utils.db.NotFoundError, ValueError): - raise click.BadParameter("Invalid schema") - - -def schema_summary(schema: dict) -> str: - """ - Extract property names from a JSON schema and format them in a - concise way that highlights the array/object structure. - - Args: - schema (dict): A JSON schema dictionary - - Returns: - str: A human-friendly summary of the schema structure - """ - if not schema or not isinstance(schema, dict): - return "" - - schema_type = schema.get("type", "") - - if schema_type == "object": - props = schema.get("properties", {}) - prop_summaries = [] - - for name, prop_schema in props.items(): - prop_type = prop_schema.get("type", "") - - if prop_type == "array": - items = prop_schema.get("items", {}) - items_summary = schema_summary(items) - prop_summaries.append(f"{name}: [{items_summary}]") - elif prop_type == "object": - nested_summary = schema_summary(prop_schema) - prop_summaries.append(f"{name}: {nested_summary}") - else: - prop_summaries.append(name) - - return "{" + ", ".join(prop_summaries) + "}" - - elif schema_type == "array": - items = schema.get("items", {}) - return schema_summary(items) - - return "" - - -def schema_dsl(schema_dsl: str, multi: bool = False) -> Dict[str, Any]: - """ - Build a JSON schema from a concise schema string. - - Args: - schema_dsl: A string representing a schema in the concise format. - Can be comma-separated or newline-separated. - multi: Boolean, return a schema for an "items" array of these - - Returns: - A dictionary representing the JSON schema. - """ - # Type mapping dictionary - type_mapping = { - "int": "integer", - "float": "number", - "bool": "boolean", - "str": "string", - } - - # Initialize the schema dictionary with required elements - json_schema: Dict[str, Any] = {"type": "object", "properties": {}, "required": []} - - # Check if the schema is newline-separated or comma-separated - if "\n" in schema_dsl: - fields = [field.strip() for field in schema_dsl.split("\n") if field.strip()] - else: - fields = [field.strip() for field in schema_dsl.split(",") if field.strip()] - - # Process each field - for field in fields: - # Extract field name, type, and description - if ":" in field: - field_info, description = field.split(":", 1) - description = description.strip() - else: - field_info = field - description = "" - - # Process field name and type - field_parts = field_info.strip().split() - field_name = field_parts[0].strip() - - # Default type is string - field_type = "string" - - # If type is specified, use it - if len(field_parts) > 1: - type_indicator = field_parts[1].strip() - if type_indicator in type_mapping: - field_type = type_mapping[type_indicator] - - # Add field to properties - json_schema["properties"][field_name] = {"type": field_type} - - # Add description if provided - if description: - json_schema["properties"][field_name]["description"] = description - - # Add field to required list - json_schema["required"].append(field_name) - - if multi: - return multi_schema(json_schema) - else: - return json_schema - - -def multi_schema(schema: dict) -> dict: - "Wrap JSON schema in an 'items': [] array" - return { - "type": "object", - "properties": {"items": {"type": "array", "items": schema}}, - "required": ["items"], - } - - -def find_unused_key(item: dict, key: str) -> str: - 'Return unused key, e.g. for {"id": "1"} and key "id" returns "id_"' - while key in item: - key += "_" - return key - - -def truncate_string( - text: str, - max_length: int = 100, - normalize_whitespace: bool = False, - keep_end: bool = False, -) -> str: - """ - Truncate a string to a maximum length, with options to normalize whitespace and keep both start and end. - - Args: - text: The string to truncate - max_length: Maximum length of the result string - normalize_whitespace: If True, replace all whitespace with a single space - keep_end: If True, keep both beginning and end of string - - Returns: - Truncated string - """ - if not text: - return text - - if normalize_whitespace: - text = re.sub(r"\s+", " ", text) - - if len(text) <= max_length: - return text - - # Minimum sensible length for keep_end is 9 characters: "a... z" - min_keep_end_length = 9 - - if keep_end and max_length >= min_keep_end_length: - # Calculate how much text to keep at each end - # Subtract 5 for the "... " separator - cutoff = (max_length - 5) // 2 - return text[:cutoff] + "... " + text[-cutoff:] - else: - # Fall back to simple truncation for very small max_length - return text[: max_length - 3] + "..." - - -def ensure_fragment(db, content): - sql = """ - insert into fragments (hash, content, datetime_utc, source) - values (:hash, :content, datetime('now'), :source) - on conflict(hash) do nothing - """ - hash_id = hashlib.sha256(content.encode("utf-8")).hexdigest() - source = None - if isinstance(content, Fragment): - source = content.source - with db.conn: - db.execute(sql, {"hash": hash_id, "content": content, "source": source}) - return list( - db.query("select id from fragments where hash = :hash", {"hash": hash_id}) - )[0]["id"] - - -def ensure_tool(db, tool): - sql = """ - insert into tools (hash, name, description, input_schema, plugin) - values (:hash, :name, :description, :input_schema, :plugin) - on conflict(hash) do nothing - """ - with db.conn: - db.execute( - sql, - { - "hash": tool.hash(), - "name": tool.name, - "description": tool.description, - "input_schema": json.dumps(tool.input_schema), - "plugin": tool.plugin, - }, - ) - return list( - db.query("select id from tools where hash = :hash", {"hash": tool.hash()}) - )[0]["id"] - - -def maybe_fenced_code(content: str) -> str: - "Return the content as a fenced code block if it looks like code" - is_code = False - if content.count("<") > 10: - is_code = True - if not is_code: - # Are 90% of the lines under 120 chars? - lines = content.splitlines() - if len(lines) > 3: - num_short = sum(1 for line in lines if len(line) < 120) - if num_short / len(lines) > 0.9: - is_code = True - if is_code: - # Find number of backticks not already present - num_backticks = 3 - while "`" * num_backticks in content: - num_backticks += 1 - # Add backticks - content = ( - "\n" - + "`" * num_backticks - + "\n" - + content.strip() - + "\n" - + "`" * num_backticks - ) - return content - - -_plugin_prefix_re = re.compile(r"^[a-zA-Z0-9_-]+:") - - -def has_plugin_prefix(value: str) -> bool: - "Check if value starts with alphanumeric prefix followed by a colon" - return bool(_plugin_prefix_re.match(value)) - - -def _parse_kwargs(arg_str: str) -> Dict[str, Any]: - """Parse key=value pairs where each value is valid JSON.""" - tokens = [] - buf = [] - depth = 0 - in_string = False - string_char = "" - escape = False - - for ch in arg_str: - if in_string: - buf.append(ch) - if escape: - escape = False - elif ch == "\\": - escape = True - elif ch == string_char: - in_string = False - else: - if ch in "\"'": - in_string = True - string_char = ch - buf.append(ch) - elif ch in "{[(": - depth += 1 - buf.append(ch) - elif ch in "}])": - depth -= 1 - buf.append(ch) - elif ch == "," and depth == 0: - tokens.append("".join(buf).strip()) - buf = [] - else: - buf.append(ch) - if buf: - tokens.append("".join(buf).strip()) - - kwargs: Dict[str, Any] = {} - for token in tokens: - if not token: - continue - if "=" not in token: - raise ValueError(f"Invalid keyword spec segment: '{token}'") - key, value_str = token.split("=", 1) - key = key.strip() - value_str = value_str.strip() - try: - value = json.loads(value_str) - except json.JSONDecodeError as e: - raise ValueError(f"Value for '{key}' is not valid JSON: {value_str}") from e - kwargs[key] = value - return kwargs - - -def instantiate_from_spec(class_map: Dict[str, Type], spec: str): - """ - Instantiate a class from a specification string with flexible argument formats. - - This function parses a specification string that defines a class name and its - constructor arguments, then instantiates the class using the provided class - mapping. The specification supports multiple argument formats for flexibility. - - Parameters - ---------- - class_map : Dict[str, Type] - A mapping from class names (strings) to their corresponding class objects. - Only classes present in this mapping can be instantiated. - spec : str - A specification string defining the class to instantiate and its arguments. - - Format: "ClassName" or "ClassName(arguments)" - - Supported argument formats: - - Empty: ClassName() - calls constructor with no arguments - - JSON object: ClassName({"key": "value", "other": 42}) - unpacked as **kwargs - - Single JSON value: ClassName("hello") or ClassName([1,2,3]) - passed as single positional argument - - Key-value pairs: ClassName(name="test", count=5, items=[1,2]) - parsed as individual kwargs - where values must be valid JSON - - Returns - ------- - object - An instance of the specified class, constructed with the parsed arguments. - - Raises - ------ - ValueError - If the spec string format is invalid, if the class name is not found in - class_map, if JSON parsing fails, or if argument parsing encounters errors. - """ - m = re.fullmatch(r"\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?:\((.*)\))?\s*$", spec) - if not m: - raise ValueError(f"Invalid spec string: '{spec}'") - class_name, arg_body = m.group(1), (m.group(2) or "").strip() - if class_name not in class_map: - raise ValueError(f"Unknown class '{class_name}'") - - cls = class_map[class_name] - - # No arguments at all - if arg_body == "": - return cls() - - # Starts with { -> JSON object to kwargs - if arg_body.lstrip().startswith("{"): - try: - kw = json.loads(arg_body) - except json.JSONDecodeError as e: - raise ValueError("Argument JSON object is not valid JSON") from e - if not isinstance(kw, dict): - raise ValueError("Top-level JSON must be an object when using {} form") - return cls(**kw) - - # Starts with quote / number / [ / t f n for single positional JSON value - if re.match(r'\s*(["\[\d\-]|true|false|null)', arg_body, re.I): - try: - positional_value = json.loads(arg_body) - except json.JSONDecodeError as e: - raise ValueError("Positional argument must be valid JSON") from e - return cls(positional_value) - - # Otherwise treat as key=value pairs - kwargs = _parse_kwargs(arg_body) - return cls(**kwargs) - - -NANOSECS_IN_MILLISECS = 1000000 -TIMESTAMP_LEN = 6 -RANDOMNESS_LEN = 10 - -_lock: Final = threading.Lock() -_last: Optional[bytes] = None # 16-byte last produced ULID - - -def monotonic_ulid() -> ULID: - """ - Return a ULID instance that is guaranteed to be *strictly larger* than every - other ULID returned by this function inside the same process. - - It works the same way the reference JavaScript `monotonicFactory` does: - * If the current call happens in the same millisecond as the previous - one, the 80-bit randomness part is incremented by exactly one. - * As soon as the system clock moves forward, a brand-new ULID with - cryptographically secure randomness is generated. - * If more than 2**80 ULIDs are requested within a single millisecond - an `OverflowError` is raised (practically impossible). - """ - global _last - - now_ms = time.time_ns() // NANOSECS_IN_MILLISECS - - with _lock: - # First call - if _last is None: - _last = _fresh(now_ms) - return ULID(_last) - - # Decode timestamp from the last ULID we handed out - last_ms = int.from_bytes(_last[:TIMESTAMP_LEN], "big") - - # If the millisecond is the same, increment the randomness - if now_ms == last_ms: - rand_int = int.from_bytes(_last[TIMESTAMP_LEN:], "big") + 1 - if rand_int >= 1 << (RANDOMNESS_LEN * 8): - raise OverflowError( - "Randomness overflow: > 2**80 ULIDs requested " - "in one millisecond!" - ) - randomness = rand_int.to_bytes(RANDOMNESS_LEN, "big") - _last = _last[:TIMESTAMP_LEN] + randomness - return ULID(_last) - - # New millisecond, start fresh - _last = _fresh(now_ms) - return ULID(_last) - - -def _fresh(ms: int) -> bytes: - """Build a brand-new 16-byte ULID for the given millisecond.""" - timestamp = int.to_bytes(ms, TIMESTAMP_LEN, "big") - randomness = os.urandom(RANDOMNESS_LEN) - return timestamp + randomness From d4f3f242ceeef638209350f34242e682f2a651bb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Apr 2026 20:17:39 -0700 Subject: [PATCH 033/258] Unwrap wrapped text in docs --- docs/python-api.md | 41 +++++++++-------------------------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/docs/python-api.md b/docs/python-api.md index 117ba6c36..26549a48d 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -530,14 +530,7 @@ If a response has been evaluated, `response.text()` will continue to return the ### Structured messages and streaming events -LLM has a structured view of a conversation that sits alongside the -simple string API. Each turn is an `llm.Message` with a `role` -(`"user"`, `"assistant"`, `"system"`, or `"tool"`) and a list of -`Part` objects — `TextPart`, `ReasoningPart`, `ToolCallPart`, -`ToolResultPart`, or `AttachmentPart`. You can pass explicit -structured input via `messages=[...]`, observe typed events as the -model streams, and inspect the assembled message after the response -completes. +LLM has a structured view of a conversation that sits alongside the simple string API. Each turn is an `llm.Message` with a `role` (`"user"`, `"assistant"`, `"system"`, or `"tool"`) and a list of `Part` objects — `TextPart`, `ReasoningPart`, `ToolCallPart`, `ToolResultPart`, or `AttachmentPart`. You can pass explicit structured input via `messages=[...]`, observe typed events as the model streams, and inspect the assembled message after the response completes. ```python import llm @@ -554,19 +547,13 @@ response = model.prompt(messages=[ print(response.text()) ``` -The `user`, `assistant`, `system`, and `tool_message` helpers accept -strings (wrapped as `TextPart`), `llm.Attachment` instances (wrapped -as `AttachmentPart`), existing `Part` objects, and nested lists or -tuples. +The `user`, `assistant`, `system`, and `tool_message` helpers accept strings (wrapped as `TextPart`), `llm.Attachment` instances (wrapped as `AttachmentPart`), existing `Part` objects, and nested lists or tuples. -The simple `model.prompt("hi", system="Be brief.")` form keeps -working — it's equivalent to -`model.prompt(messages=[system("Be brief."), user("hi")])`. +The simple `model.prompt("hi", system="Be brief.")` form is equivalent to `model.prompt(messages=[system("Be brief."), user("hi")])`. #### Streaming events as they arrive -`response.stream_events()` yields typed events for every content -block the model produces, live: +`response.stream_events()` yields typed events for every content block the model produces as they stream in. This is useful for interfaces that show the model response "live". ```python response = model.prompt("Explain quantum computing briefly.") @@ -581,20 +568,13 @@ for event in response.stream_events(): print(event.chunk, end="", flush=True) ``` -Event types are `"text"`, `"reasoning"`, `"tool_call_name"`, -`"tool_call_args"`, and `"tool_result"`. Each event carries a -`part_index` that groups events into the same logical Part (all -events at the same `part_index` assemble into one Part after the -stream completes). For async models, use -`async for event in response.astream_events()`. +Event types are `"text"`, `"reasoning"`, `"tool_call_name"`, `"tool_call_args"`, and `"tool_result"`. Each event carries a `part_index` that groups events into the same logical Part (all events at the same `part_index` assemble into one Part after the stream completes). For async models, use `async for event in response.astream_events()`. -Plain iteration (`for chunk in response`) continues to yield only -text strings — reasoning and tool-call events are filtered out. +Plain iteration (`for chunk in response`) continues to yield only text strings — reasoning and tool-call events are filtered out. #### Inspecting the finished response -After a response completes, `response.messages` gives you the -assembled list of `Message` objects: +After a response completes, `response.messages` gives you the assembled list of `Message` objects: ```python response = model.prompt("What's 2+2?") @@ -606,9 +586,7 @@ for message in response.messages: #### Persisting a conversation yourself -Messages and Parts round-trip through plain Python dicts via -`to_dict()` / `from_dict()`, so your application can persist -conversations to any JSON-capable store without touching SQLite: +Messages and Parts round-trip through plain Python dicts via `to_dict()` / `from_dict()`, so your application can persist conversations to any JSON-capable store without touching SQLite: ```python import json @@ -630,8 +608,7 @@ response = model.prompt(messages=rebuilt + [user("And 3+3?")]) print(response.text()) ``` -`AttachmentPart` bytes are base64-encoded in the dict form, so full -multi-modal conversations round-trip faithfully. +`AttachmentPart` bytes are base64-encoded in the dict form, so full multi-modal conversations round-trip faithfully. (python-api-async)= From 65b8e37c797d3d997716ccf9731d09490d695e53 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Apr 2026 21:10:13 -0700 Subject: [PATCH 034/258] Ran Black --- llm/default_plugins/openai_models.py | 36 ++---- llm/models.py | 36 ++---- llm/parts.py | 10 +- llm/serialization.py | 1 - tests/test_async_parity.py | 35 +++--- tests/test_cli_streaming.py | 12 +- tests/test_openai_messages.py | 45 ++----- tests/test_parts.py | 182 ++++++++++----------------- tests/test_serialization.py | 76 ++++++----- 9 files changed, 170 insertions(+), 263 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 9a7013390..804655329 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -722,9 +722,7 @@ def build_messages(self, prompt, conversation): messages: List[Dict[str, Any]] = [] current_system: Optional[str] = None for msg in prompt.messages: - current_system = self._append_llm_message( - messages, msg, current_system - ) + current_system = self._append_llm_message(messages, msg, current_system) return messages def set_usage(self, response, usage): @@ -851,9 +849,9 @@ def execute( tool_call_id=tool_call.id, ) else: - tool_calls[idx].function.arguments += ( - tool_call.function.arguments - ) + tool_calls[ + idx + ].function.arguments += tool_call.function.arguments if tool_call.function.arguments: yield StreamEvent( type="tool_call_args", @@ -869,9 +867,7 @@ def execute( # Empty strings are noise (OpenAI's first chunk # with role=assistant has content=""). seen_text = True - yield StreamEvent( - type="text", chunk=content, part_index=0 - ) + yield StreamEvent(type="text", chunk=content, part_index=0) response.response_json = remove_dict_none_values(combine_chunks(chunks)) if tool_calls: for value in tool_calls.values(): @@ -923,10 +919,8 @@ def execute( # set_usage pops top-level keys and passes the rest through # simplify_usage_dict, which strips zero-valued entries. if usage: - reasoning_tokens = ( - (usage.get("completion_tokens_details") or {}).get( - "reasoning_tokens", 0 - ) + reasoning_tokens = (usage.get("completion_tokens_details") or {}).get( + "reasoning_tokens", 0 ) if reasoning_tokens: response._reasoning_token_count = reasoning_tokens @@ -990,9 +984,9 @@ async def execute( tool_call_id=tool_call.id, ) else: - tool_calls[idx].function.arguments += ( - tool_call.function.arguments - ) + tool_calls[ + idx + ].function.arguments += tool_call.function.arguments if tool_call.function.arguments: yield StreamEvent( type="tool_call_args", @@ -1005,9 +999,7 @@ async def execute( except IndexError: content = None if content: - yield StreamEvent( - type="text", chunk=content, part_index=0 - ) + yield StreamEvent(type="text", chunk=content, part_index=0) if tool_calls: for value in tool_calls.values(): response.add_tool_call( @@ -1057,10 +1049,8 @@ async def execute( ) # See sync Chat.execute: capture reasoning before set_usage mutates. if usage: - reasoning_tokens = ( - (usage.get("completion_tokens_details") or {}).get( - "reasoning_tokens", 0 - ) + reasoning_tokens = (usage.get("completion_tokens_details") or {}).get( + "reasoning_tokens", 0 ) if reasoning_tokens: response._reasoning_token_count = reasoning_tokens diff --git a/llm/models.py b/llm/models.py index 8600eb402..541ed5743 100644 --- a/llm/models.py +++ b/llm/models.py @@ -437,9 +437,7 @@ def messages(self): result: List["Message"] = [] if self.system: - result.append( - Message(role="system", parts=[TextPart(text=self.system)]) - ) + result.append(Message(role="system", parts=[TextPart(text=self.system)])) if self.tool_results: result.append( @@ -948,9 +946,7 @@ def _build_parts(self) -> List[Any]: tool_call_id=tc.tool_call_id, ) ) - reasoning_token_count = getattr( - self, "_reasoning_token_count", 0 - ) + reasoning_token_count = getattr(self, "_reasoning_token_count", 0) if reasoning_token_count: parts.insert( 0, @@ -989,9 +985,7 @@ def finalize(): elif current_family == "reasoning": text = "".join(text_buf) if text: - parts.append( - ReasoningPart(text=text, provider_metadata=pm_merged) - ) + parts.append(ReasoningPart(text=text, provider_metadata=pm_merged)) elif current_family == "tool_call": args_str = "".join(tool_args_buf) try: @@ -1464,12 +1458,8 @@ def _response_from_dict( model = getter(data["model"]) prompt_data = data.get("prompt", {}) - input_messages = [ - Message.from_dict(m) for m in prompt_data.get("messages", []) - ] - output_messages = [ - Message.from_dict(m) for m in data.get("messages", []) - ] + input_messages = [Message.from_dict(m) for m in prompt_data.get("messages", [])] + output_messages = [Message.from_dict(m) for m in data.get("messages", [])] options_kwargs = prompt_data.get("options") or {} system = prompt_data.get("system") @@ -1538,9 +1528,7 @@ def reply( self._force() chain: List[Any] = list(self.prompt.messages) + list(self.messages) if prompt: - chain.append( - Message(role="user", parts=[TextPart(text=prompt)]) - ) + chain.append(Message(role="user", parts=[TextPart(text=prompt)])) if messages: chain.extend(messages) return self.model.prompt(messages=chain, **kwargs) @@ -1853,9 +1841,7 @@ def reply( ) chain: List[Any] = list(self.prompt.messages) + list(self.messages) if prompt: - chain.append( - Message(role="user", parts=[TextPart(text=prompt)]) - ) + chain.append(Message(role="user", parts=[TextPart(text=prompt)])) if messages: chain.extend(messages) return self.model.prompt(messages=chain, **kwargs) @@ -2154,9 +2140,7 @@ def messages(self) -> List[Any]: if loaded is not None: return list(loaded) if not self._done: - raise ValueError( - "Response not yet awaited — use 'await response' first" - ) + raise ValueError("Response not yet awaited — use 'await response' first") parts = self._build_parts() if not parts: return [] @@ -2286,9 +2270,7 @@ def __repr__(self): return "".format(self.prompt.prompt, text) -def _chain_for_tool_results( - prior_response, tool_results, attachments -) -> List[Any]: +def _chain_for_tool_results(prior_response, tool_results, attachments) -> List[Any]: """Build the message chain for a tool-result turn in a chain loop. Takes the prior response's full input chain + its structured diff --git a/llm/parts.py b/llm/parts.py index e627f876b..3c12f1ef1 100644 --- a/llm/parts.py +++ b/llm/parts.py @@ -267,9 +267,7 @@ def normalize_parts(items: Any) -> List[Part]: return out -def system( - *items: Any, provider_metadata: Optional[Dict[str, Any]] = None -) -> Message: +def system(*items: Any, provider_metadata: Optional[Dict[str, Any]] = None) -> Message: "Build a Message with role='system'." return Message( role="system", @@ -278,9 +276,7 @@ def system( ) -def user( - *items: Any, provider_metadata: Optional[Dict[str, Any]] = None -) -> Message: +def user(*items: Any, provider_metadata: Optional[Dict[str, Any]] = None) -> Message: "Build a Message with role='user'." return Message( role="user", @@ -330,7 +326,7 @@ class StreamEvent: """ type: str # "text" / "reasoning" / "tool_call_name" / - # "tool_call_args" / "tool_result" + # "tool_call_args" / "tool_result" chunk: str part_index: int tool_call_id: Optional[str] = None diff --git a/llm/serialization.py b/llm/serialization.py index b6b4bf972..870542ae4 100644 --- a/llm/serialization.py +++ b/llm/serialization.py @@ -38,7 +38,6 @@ def save_messages(conn, messages: list[MessageDict]) -> None: # support. typing_extensions is a transitive dep via pydantic. from typing_extensions import NotRequired, TypedDict - __all__ = [ "AttachmentDict", "AttachmentPartDict", diff --git a/tests/test_async_parity.py b/tests/test_async_parity.py index 981fbcdcf..5c2479a6f 100644 --- a/tests/test_async_parity.py +++ b/tests/test_async_parity.py @@ -11,7 +11,6 @@ import llm import pytest - # ---- basic sanity: both variants are registered -------------------- @@ -202,12 +201,12 @@ async def my_tool(x: int) -> int: # Drive a one-iteration chain by asking echo to emit a tool call # (echo's JSON-prompt syntax). chain = model.chain( - json.dumps({ - "tool_calls": [ - {"name": "my_tool", "arguments": {"x": 5}} - ], - "prompt": "prompt", - }), + json.dumps( + { + "tool_calls": [{"name": "my_tool", "arguments": {"x": 5}}], + "prompt": "prompt", + } + ), tools=[llm.Tool.function(my_tool, name="my_tool")], ) @@ -267,9 +266,7 @@ async def test_async_from_dict_model_override(): # Pass model explicitly to override whatever's in the payload. alt = llm.get_async_model("echo") - restored = llm.AsyncResponse.from_dict( - json.loads(payload), model=alt - ) + restored = llm.AsyncResponse.from_dict(json.loads(payload), model=alt) assert restored.model is alt @@ -350,7 +347,11 @@ async def test_async_full_chain_to_dict_round_trip_three_turns(): payload = json.dumps(r3.to_dict()) restored = llm.AsyncResponse.from_dict(json.loads(payload)) assert [m.role for m in restored.prompt.messages] == [ - "user", "assistant", "user", "assistant", "user" + "user", + "assistant", + "user", + "assistant", + "user", ] texts = [m.parts[0].text for m in restored.prompt.messages if m.parts] assert texts[0] == "q1" @@ -361,7 +362,13 @@ async def test_async_full_chain_to_dict_round_trip_three_turns(): r4 = restored.reply("q4") await r4.text() assert [m.role for m in r4.prompt.messages] == [ - "user", "assistant", "user", "assistant", "user", "assistant", "user" + "user", + "assistant", + "user", + "assistant", + "user", + "assistant", + "user", ] @@ -376,9 +383,7 @@ async def test_async_reply_chains_three_turns(): await r3.text() chain = r3.prompt.messages - assert [m.role for m in chain] == [ - "user", "assistant", "user", "assistant", "user" - ] + assert [m.role for m in chain] == ["user", "assistant", "user", "assistant", "user"] texts = [m.parts[0].text for m in chain if m.parts] assert texts[0] == "q1" assert texts[2] == "q2" diff --git a/tests/test_cli_streaming.py b/tests/test_cli_streaming.py index f631f1d91..c6ad624e7 100644 --- a/tests/test_cli_streaming.py +++ b/tests/test_cli_streaming.py @@ -26,9 +26,7 @@ def test_text_goes_to_stdout_not_stderr(mock_model): def test_reasoning_goes_to_stderr_not_stdout(mock_model): mock_model.enqueue( [ - llm.StreamEvent( - type="reasoning", chunk="thinking hard", part_index=0 - ), + llm.StreamEvent(type="reasoning", chunk="thinking hard", part_index=0), llm.StreamEvent(type="text", chunk="answer", part_index=1), ] ) @@ -68,9 +66,7 @@ def test_reasoning_rendered_in_dim_style(mock_model): def test_no_reasoning_flag_suppresses_reasoning(mock_model): mock_model.enqueue( [ - llm.StreamEvent( - type="reasoning", chunk="hidden thinking", part_index=0 - ), + llm.StreamEvent(type="reasoning", chunk="hidden thinking", part_index=0), llm.StreamEvent(type="text", chunk="answer", part_index=1), ] ) @@ -126,9 +122,7 @@ def test_newline_between_reasoning_and_text(mock_model): def test_async_path_reasoning_to_stderr(async_mock_model): async_mock_model.enqueue( [ - llm.StreamEvent( - type="reasoning", chunk="async thinking", part_index=0 - ), + llm.StreamEvent(type="reasoning", chunk="async thinking", part_index=0), llm.StreamEvent(type="text", chunk="async answer", part_index=1), ] ) diff --git a/tests/test_openai_messages.py b/tests/test_openai_messages.py index ca2fc0270..459633a1e 100644 --- a/tests/test_openai_messages.py +++ b/tests/test_openai_messages.py @@ -17,7 +17,6 @@ from llm.default_plugins.openai_models import Chat from llm.models import Prompt - API_KEY = "badkey" @@ -27,9 +26,7 @@ def _sse(delta, finish_reason=None, usage=None, tool_calls=None): "object": "chat.completion.chunk", "created": 1700000000, "model": "gpt-4o-mini", - "choices": [ - {"index": 0, "delta": delta, "finish_reason": finish_reason} - ], + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], } if tool_calls is not None: chunk["choices"][0]["delta"]["tool_calls"] = tool_calls @@ -111,9 +108,7 @@ def chat_model(): class TestBuildMessagesFromExplicitMessages: def test_single_user_message(self, chat_model): - prompt = Prompt( - None, model=chat_model, messages=[llm.user("hi")] - ) + prompt = Prompt(None, model=chat_model, messages=[llm.user("hi")]) result = chat_model.build_messages(prompt, None) assert result == [{"role": "user", "content": "hi"}] @@ -130,9 +125,7 @@ def test_system_plus_user(self, chat_model): ] def test_user_with_attachment(self, chat_model): - att = llm.Attachment( - type="image/jpeg", url="http://example.com/cat.jpg" - ) + att = llm.Attachment(type="image/jpeg", url="http://example.com/cat.jpg") prompt = Prompt( None, model=chat_model, @@ -213,9 +206,7 @@ def test_assistant_tool_call_only_no_text(self, chat_model): } def test_tool_role_message_with_tool_result(self, chat_model): - tr = llm.ToolResultPart( - name="search", output="sunny", tool_call_id="c1" - ) + tr = llm.ToolResultPart(name="search", output="sunny", tool_call_id="c1") prompt = Prompt( None, model=chat_model, @@ -430,9 +421,7 @@ def test_text_stream_yields_text_events(self, httpx_mock): # Text chunks concatenate to the expected full text. assert "".join(e.chunk for e in events) == "Hello" - def test_text_stream_plain_iteration_still_returns_strings( - self, httpx_mock - ): + def test_text_stream_plain_iteration_still_returns_strings(self, httpx_mock): httpx_mock.add_response( method="POST", url="https://api.openai.com/v1/chat/completions", @@ -456,9 +445,7 @@ def test_text_stream_messages_assembled(self, httpx_mock): response = model.prompt("hi", key=API_KEY) response.text() assert response.messages == [ - llm.Message( - role="assistant", parts=[llm.TextPart(text="Hello")] - ) + llm.Message(role="assistant", parts=[llm.TextPart(text="Hello")]) ] def test_tool_call_stream_yields_name_and_args_events(self, httpx_mock): @@ -487,9 +474,7 @@ def get_weather(city: str) -> str: # valid JSON. args_events = [e for e in events if e.type == "tool_call_args"] assert all(e.part_index == name_ev.part_index for e in args_events) - assert json.loads("".join(e.chunk for e in args_events)) == { - "city": "Paris" - } + assert json.loads("".join(e.chunk for e in args_events)) == {"city": "Paris"} def test_tool_call_registered_via_add_tool_call(self, httpx_mock): """response.tool_calls() still works — chain/execute relies on it.""" @@ -572,9 +557,7 @@ def _text_stream_with_reasoning_usage(reasoning_tokens): "prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7, - "completion_tokens_details": { - "reasoning_tokens": reasoning_tokens - }, + "completion_tokens_details": {"reasoning_tokens": reasoning_tokens}, }, ) yield b"data: [DONE]\n\n" @@ -607,9 +590,7 @@ def test_reasoning_part_prepended_to_messages(self, httpx_mock): llm.Message( role="assistant", parts=[ - llm.ReasoningPart( - text="", redacted=True, token_count=150 - ), + llm.ReasoningPart(text="", redacted=True, token_count=150), llm.TextPart(text="Hello"), ], ) @@ -657,11 +638,7 @@ def test_non_streaming_text_yields_single_event(self, httpx_mock): model = llm.get_model("gpt-4o-mini") response = model.prompt("hi", key=API_KEY, stream=False) events = list(response.stream_events()) - assert events == [ - llm.StreamEvent(type="text", chunk="Hello", part_index=0) - ] + assert events == [llm.StreamEvent(type="text", chunk="Hello", part_index=0)] assert response.messages == [ - llm.Message( - role="assistant", parts=[llm.TextPart(text="Hello")] - ) + llm.Message(role="assistant", parts=[llm.TextPart(text="Hello")]) ] diff --git a/tests/test_parts.py b/tests/test_parts.py index 9cee6b672..779f221a7 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -9,7 +9,6 @@ import llm - # -- Exports ------------------------------------------------------------ @@ -46,9 +45,7 @@ def test_to_dict_shape(self): assert llm.TextPart(text="hi").to_dict() == {"type": "text", "text": "hi"} def test_with_provider_metadata(self): - part = llm.TextPart( - text="hi", provider_metadata={"openai": {"flag": True}} - ) + part = llm.TextPart(text="hi", provider_metadata={"openai": {"flag": True}}) restored = llm.Part.from_dict(part.to_dict()) assert restored == part @@ -97,9 +94,7 @@ def test_server_executed_flag_roundtrips(self): class TestToolResultPart: def test_roundtrip(self): - part = llm.ToolResultPart( - name="search", output="72F sunny", tool_call_id="c1" - ) + part = llm.ToolResultPart(name="search", output="72F sunny", tool_call_id="c1") restored = llm.Part.from_dict(part.to_dict()) assert restored == part assert restored.exception is None @@ -523,9 +518,7 @@ def execute(self, prompt, stream, response, conversation): response = m.prompt("x") response.text() parts = response.messages[0].parts - assert parts[0] == llm.ReasoningPart( - text="", redacted=True, token_count=200 - ) + assert parts[0] == llm.ReasoningPart(text="", redacted=True, token_count=200) assert parts[1] == llm.TextPart(text="hi") @@ -551,9 +544,7 @@ def test_events_arrive_before_done(self, mock_model): assert response._done def test_stream_events_after_done_replays(self, mock_model): - mock_model.enqueue( - [llm.StreamEvent(type="text", chunk="hi", part_index=0)] - ) + mock_model.enqueue([llm.StreamEvent(type="text", chunk="hi", part_index=0)]) response = mock_model.prompt("x") first = list(response.stream_events()) # Second call replays from the stored events. @@ -612,9 +603,7 @@ async def test_async_messages_after_await(self, async_mock_model): response = async_mock_model.prompt("x") await response.text() assert response.messages == [ - llm.Message( - role="assistant", parts=[llm.TextPart(text="hi")] - ) + llm.Message(role="assistant", parts=[llm.TextPart(text="hi")]) ] @@ -635,9 +624,7 @@ def test_prompt_text_synthesizes_user_message(self, mock_model): from llm.models import Prompt p = Prompt("hi", model=mock_model) - assert p.messages == [ - llm.Message(role="user", parts=[llm.TextPart(text="hi")]) - ] + assert p.messages == [llm.Message(role="user", parts=[llm.TextPart(text="hi")])] def test_system_and_prompt_synthesizes_two_messages(self, mock_model): from llm.models import Prompt @@ -672,11 +659,7 @@ def test_tool_results_become_tool_role_message(self, mock_model): assert p.messages == [ llm.Message( role="tool", - parts=[ - llm.ToolResultPart( - name="t", output="ok", tool_call_id="c1" - ) - ], + parts=[llm.ToolResultPart(name="t", output="ok", tool_call_id="c1")], ) ] @@ -694,9 +677,7 @@ def test_explicit_messages_returned_verbatim(self, mock_model): p = Prompt(None, model=mock_model, messages=explicit) assert p.messages == explicit - def test_explicit_messages_ignores_prompt_kwarg( - self, mock_model - ): + def test_explicit_messages_ignores_prompt_kwarg(self, mock_model): """Explicit messages= is authoritative. A prompt= string passed alongside is no longer auto-appended — the invariant is that prompt.messages equals exactly what the model was sent.""" @@ -728,9 +709,7 @@ def test_model_prompt_accepts_messages(self, mock_model): def test_model_prompt_messages_with_system(self, mock_model): mock_model.enqueue(["ok"]) - response = mock_model.prompt( - messages=[llm.system("be brief"), llm.user("hi")] - ) + response = mock_model.prompt(messages=[llm.system("be brief"), llm.user("hi")]) response.text() assert response.prompt.messages == [ llm.system("be brief"), @@ -752,9 +731,7 @@ async def test_async_model_prompt_accepts_messages(self, async_mock_model): assert response.prompt.messages == [llm.user("hi")] @pytest.mark.asyncio - async def test_async_conversation_prompt_accepts_messages( - self, async_mock_model - ): + async def test_async_conversation_prompt_accepts_messages(self, async_mock_model): async_mock_model.enqueue(["ok"]) conv = async_mock_model.conversation() response = conv.prompt(messages=[llm.user("q")]) @@ -783,9 +760,7 @@ def test_explicit_messages_is_authoritative_no_prompt_combine(self, mock_model): response.text() assert response.prompt.messages == [llm.user("q")] - def test_conversation_second_turn_prompt_messages_has_full_chain( - self, mock_model - ): + def test_conversation_second_turn_prompt_messages_has_full_chain(self, mock_model): mock_model.enqueue(["a1"]) mock_model.enqueue(["a2"]) conv = mock_model.conversation() @@ -802,16 +777,17 @@ def test_conversation_second_turn_prompt_messages_has_full_chain( llm.user("q2"), ] - def test_conversation_third_turn_includes_everything_before( - self, mock_model - ): + def test_conversation_third_turn_includes_everything_before(self, mock_model): mock_model.enqueue(["a1"]) mock_model.enqueue(["a2"]) mock_model.enqueue(["a3"]) conv = mock_model.conversation() - r1 = conv.prompt("q1"); r1.text() - r2 = conv.prompt("q2"); r2.text() - r3 = conv.prompt("q3"); r3.text() + r1 = conv.prompt("q1") + r1.text() + r2 = conv.prompt("q2") + r2.text() + r3 = conv.prompt("q3") + r3.text() assert r3.prompt.messages == [ llm.user("q1"), @@ -821,25 +797,23 @@ def test_conversation_third_turn_includes_everything_before( llm.user("q3"), ] - def test_conversation_first_turn_chain_is_single_user_message( - self, mock_model - ): + def test_conversation_first_turn_chain_is_single_user_message(self, mock_model): mock_model.enqueue(["a1"]) conv = mock_model.conversation() r1 = conv.prompt("q1") r1.text() assert r1.prompt.messages == [llm.user("q1")] - def test_conversation_preserves_reasoning_and_tool_call_parts( - self, mock_model - ): + def test_conversation_preserves_reasoning_and_tool_call_parts(self, mock_model): """The chain carries reasoning and tool calls from prior turns, not just the flat text — required for multi-turn extended thinking (Claude) and tool-use round-trips.""" - mock_model.enqueue([ - llm.StreamEvent(type="reasoning", chunk="thinking...", part_index=0), - llm.StreamEvent(type="text", chunk="answer", part_index=1), - ]) + mock_model.enqueue( + [ + llm.StreamEvent(type="reasoning", chunk="thinking...", part_index=0), + llm.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) mock_model.enqueue(["follow-up answer"]) conv = mock_model.conversation() r1 = conv.prompt("q1") @@ -962,9 +936,12 @@ def test_reply_chains(self, mock_model): mock_model.enqueue(["a1"]) mock_model.enqueue(["a2"]) mock_model.enqueue(["a3"]) - r1 = mock_model.prompt("q1"); r1.text() - r2 = r1.reply("q2"); r2.text() - r3 = r2.reply("q3"); r3.text() + r1 = mock_model.prompt("q1") + r1.text() + r2 = r1.reply("q2") + r2.text() + r3 = r2.reply("q3") + r3.text() assert r3.prompt.messages == [ llm.user("q1"), llm.assistant("a1"), @@ -1028,9 +1005,7 @@ class TestChainPropagatesSystem: def test_sync_chain_tool_result_turn_preserves_system(self, mock_model): # First turn: fake a tool call so the chain iterates. - tool_call = llm.ToolCall( - tool_call_id="c1", name="tick", arguments={} - ) + tool_call = llm.ToolCall(tool_call_id="c1", name="tick", arguments={}) class ChainMock(type(mock_model)): def execute(self, prompt, stream, response, conversation): @@ -1057,12 +1032,8 @@ def tick() -> str: second = chain._responses[1] assert second.prompt.system == "be brief" - def test_sync_chain_tool_result_turn_preserves_system_fragments( - self, mock_model - ): - tool_call = llm.ToolCall( - tool_call_id="c1", name="tick", arguments={} - ) + def test_sync_chain_tool_result_turn_preserves_system_fragments(self, mock_model): + tool_call = llm.ToolCall(tool_call_id="c1", name="tick", arguments={}) class ChainMock(type(mock_model)): def execute(self, prompt, stream, response, conversation): @@ -1101,9 +1072,7 @@ def tick() -> str: async def test_async_chain_tool_result_turn_preserves_system( self, async_mock_model ): - tool_call = llm.ToolCall( - tool_call_id="c1", name="tick", arguments={} - ) + tool_call = llm.ToolCall(tool_call_id="c1", name="tick", arguments={}) class AsyncChainMock(type(async_mock_model)): supports_tools = True @@ -1153,9 +1122,7 @@ def test_model_chain_accepts_messages(self, mock_model): r1 = chain._responses[0] assert r1.prompt.messages == [llm.user("explicit")] - def test_chain_messages_is_authoritative_over_prompt_kwarg( - self, mock_model - ): + def test_chain_messages_is_authoritative_over_prompt_kwarg(self, mock_model): """Parity with prompt(): when both are passed, messages= wins and the prompt= string is not folded into the chain.""" mock_model.enqueue(["ok"]) @@ -1181,14 +1148,10 @@ def test_chain_with_messages_and_prior_conversation(self, mock_model): chain = conv.chain(messages=[llm.user("fresh start")]) chain.text() first_chain_response = chain._responses[0] - assert first_chain_response.prompt.messages == [ - llm.user("fresh start") - ] + assert first_chain_response.prompt.messages == [llm.user("fresh start")] @pytest.mark.asyncio - async def test_async_conversation_chain_accepts_messages( - self, async_mock_model - ): + async def test_async_conversation_chain_accepts_messages(self, async_mock_model): async_mock_model.enqueue(["ok"]) conv = async_mock_model.conversation() chain = conv.chain(messages=[llm.user("explicit")]) @@ -1197,9 +1160,7 @@ async def test_async_conversation_chain_accepts_messages( assert r1.prompt.messages == [llm.user("explicit")] @pytest.mark.asyncio - async def test_async_model_chain_accepts_messages( - self, async_mock_model - ): + async def test_async_model_chain_accepts_messages(self, async_mock_model): async_mock_model.enqueue(["ok"]) chain = async_mock_model.chain(messages=[llm.user("explicit")]) await chain.text() @@ -1253,15 +1214,17 @@ def test_from_dict_then_reply_continues_conversation(self, mock_model): ] def test_to_dict_preserves_reasoning_and_signatures(self, mock_model): - mock_model.enqueue([ - llm.StreamEvent( - type="reasoning", - chunk="thinking...", - part_index=0, - provider_metadata={"anthropic": {"signature": "sig-abc"}}, - ), - llm.StreamEvent(type="text", chunk="answer", part_index=1), - ]) + mock_model.enqueue( + [ + llm.StreamEvent( + type="reasoning", + chunk="thinking...", + part_index=0, + provider_metadata={"anthropic": {"signature": "sig-abc"}}, + ), + llm.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) r = mock_model.prompt("q") r.text() @@ -1276,21 +1239,21 @@ def test_to_dict_preserves_reasoning_and_signatures(self, mock_model): "anthropic": {"signature": "sig-abc"} } - def test_from_dict_reply_includes_prior_reasoning_in_chain( - self, mock_model - ): + def test_from_dict_reply_includes_prior_reasoning_in_chain(self, mock_model): """The thing this entire refactor was about: a reply() after from_dict() sends the thinking signature back to the model for multi-turn extended thinking.""" - mock_model.enqueue([ - llm.StreamEvent( - type="reasoning", - chunk="thinking...", - part_index=0, - provider_metadata={"anthropic": {"signature": "sig-xyz"}}, - ), - llm.StreamEvent(type="text", chunk="answer", part_index=1), - ]) + mock_model.enqueue( + [ + llm.StreamEvent( + type="reasoning", + chunk="thinking...", + part_index=0, + provider_metadata={"anthropic": {"signature": "sig-xyz"}}, + ), + llm.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) mock_model.enqueue(["a2"]) r1 = mock_model.prompt("q1") r1.text() @@ -1303,8 +1266,7 @@ def test_from_dict_reply_includes_prior_reasoning_in_chain( # The signature must be in the chain sent to the model. chain = r2.prompt.messages reasoning_parts = [ - p for m in chain for p in m.parts - if isinstance(p, llm.ReasoningPart) + p for m in chain for p in m.parts if isinstance(p, llm.ReasoningPart) ] assert len(reasoning_parts) == 1 assert reasoning_parts[0].provider_metadata == { @@ -1335,14 +1297,10 @@ def test_message_from_dict_static_method_unchanged(self): class TestChainResponseStreamEvents: - def test_sync_chain_stream_events_yields_text_when_no_tools( - self, mock_model - ): + def test_sync_chain_stream_events_yields_text_when_no_tools(self, mock_model): # Chain with no tool calls is a single-response chain — its # stream_events should concatenate from each underlying response. - mock_model.enqueue( - [llm.StreamEvent(type="text", chunk="done", part_index=0)] - ) + mock_model.enqueue([llm.StreamEvent(type="text", chunk="done", part_index=0)]) chain = mock_model.conversation().chain("q") events = list(chain.stream_events()) assert [e.type for e in events] == ["text"] @@ -1397,15 +1355,11 @@ def test_rebuilt_messages_reach_plugin_via_prompt(self, mock_model): # Later — rebuild from the wire form and continue. rebuilt = [llm.Message.from_dict(d) for d in json.loads(payload)] mock_model.enqueue(["turn 2 answer"]) - r2 = mock_model.prompt( - messages=rebuilt + [llm.user("turn 2 question")] - ) + r2 = mock_model.prompt(messages=rebuilt + [llm.user("turn 2 question")]) r2.text() # The plugin saw the full structured history on prompt.messages. - assert r2.prompt.messages == rebuilt + [ - llm.user("turn 2 question") - ] + assert r2.prompt.messages == rebuilt + [llm.user("turn 2 question")] assert r2.messages == [llm.assistant("turn 2 answer")] def test_roundtrip_preserves_tool_calls_and_results(self, mock_model): diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 9f3313bfd..73dcec6b5 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -25,7 +25,6 @@ UsageDict, ) - # ---- required/optional keys ---------------------------------------- @@ -41,30 +40,34 @@ def test_text_part_dict_required_keys(self): def test_reasoning_part_dict_required_keys(self): assert ReasoningPartDict.__required_keys__ == {"type", "text"} assert ReasoningPartDict.__optional_keys__ == { - "redacted", "token_count", "provider_metadata" + "redacted", + "token_count", + "provider_metadata", } def test_tool_call_part_dict_required_keys(self): - assert ToolCallPartDict.__required_keys__ == { - "type", "name", "arguments" - } + assert ToolCallPartDict.__required_keys__ == {"type", "name", "arguments"} assert ToolCallPartDict.__optional_keys__ == { - "tool_call_id", "server_executed", "provider_metadata" + "tool_call_id", + "server_executed", + "provider_metadata", } def test_tool_result_part_dict_required_keys(self): - assert ToolResultPartDict.__required_keys__ == { - "type", "name", "output" - } + assert ToolResultPartDict.__required_keys__ == {"type", "name", "output"} assert ToolResultPartDict.__optional_keys__ == { - "tool_call_id", "server_executed", "exception", - "attachments", "provider_metadata", + "tool_call_id", + "server_executed", + "exception", + "attachments", + "provider_metadata", } def test_attachment_part_dict_required_keys(self): assert AttachmentPartDict.__required_keys__ == {"type"} assert AttachmentPartDict.__optional_keys__ == { - "attachment", "provider_metadata" + "attachment", + "provider_metadata", } def test_response_dict_required_keys(self): @@ -90,9 +93,7 @@ def test_text_part_with_provider_metadata_matches(self): self._adapter(TextPartDict).validate_python(d) def test_reasoning_part_redacted_matches(self): - d = llm.ReasoningPart( - text="", redacted=True, token_count=150 - ).to_dict() + d = llm.ReasoningPart(text="", redacted=True, token_count=150).to_dict() self._adapter(ReasoningPartDict).validate_python(d) def test_reasoning_part_with_signature_matches(self): @@ -135,15 +136,11 @@ def test_reasoning_part_validates_as_part_dict(self): TypeAdapter(PartDict).validate_python(d) def test_tool_call_part_validates_as_part_dict(self): - d = llm.ToolCallPart( - name="t", arguments={}, tool_call_id="c1" - ).to_dict() + d = llm.ToolCallPart(name="t", arguments={}, tool_call_id="c1").to_dict() TypeAdapter(PartDict).validate_python(d) def test_tool_result_part_validates_as_part_dict(self): - d = llm.ToolResultPart( - name="t", output="out", tool_call_id="c1" - ).to_dict() + d = llm.ToolResultPart(name="t", output="out", tool_call_id="c1").to_dict() TypeAdapter(PartDict).validate_python(d) def test_attachment_part_validates_as_part_dict(self): @@ -153,9 +150,7 @@ def test_attachment_part_validates_as_part_dict(self): def test_unknown_type_rejected(self): with pytest.raises(Exception): - TypeAdapter(PartDict).validate_python( - {"type": "nonsense", "text": "x"} - ) + TypeAdapter(PartDict).validate_python({"type": "nonsense", "text": "x"}) class TestMessageDictRoundTrip: @@ -198,15 +193,17 @@ def test_mock_response_to_dict_matches(self, mock_model): TypeAdapter(ResponseDict).validate_python(d) def test_response_with_reasoning_matches(self, mock_model): - mock_model.enqueue([ - llm.StreamEvent( - type="reasoning", - chunk="thinking", - part_index=0, - provider_metadata={"anthropic": {"signature": "s"}}, - ), - llm.StreamEvent(type="text", chunk="answer", part_index=1), - ]) + mock_model.enqueue( + [ + llm.StreamEvent( + type="reasoning", + chunk="thinking", + part_index=0, + provider_metadata={"anthropic": {"signature": "s"}}, + ), + llm.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) r = mock_model.prompt("q") r.text() @@ -232,27 +229,32 @@ class TestLiteralDiscriminators: def test_text_part_literal_is_text(self): import typing + hints = typing.get_type_hints(TextPartDict) # Literal["text"] — check the args assert typing.get_args(hints["type"]) == ("text",) def test_reasoning_part_literal_is_reasoning(self): import typing + hints = typing.get_type_hints(ReasoningPartDict) assert typing.get_args(hints["type"]) == ("reasoning",) def test_tool_call_part_literal_is_tool_call(self): import typing + hints = typing.get_type_hints(ToolCallPartDict) assert typing.get_args(hints["type"]) == ("tool_call",) def test_tool_result_part_literal_is_tool_result(self): import typing + hints = typing.get_type_hints(ToolResultPartDict) assert typing.get_args(hints["type"]) == ("tool_result",) def test_attachment_part_literal_is_attachment(self): import typing + hints = typing.get_type_hints(AttachmentPartDict) assert typing.get_args(hints["type"]) == ("attachment",) @@ -265,41 +267,49 @@ class TestAnnotations: def test_text_part_to_dict_annotation(self): import typing + hints = typing.get_type_hints(llm.TextPart.to_dict) assert hints["return"] is TextPartDict def test_reasoning_part_to_dict_annotation(self): import typing + hints = typing.get_type_hints(llm.ReasoningPart.to_dict) assert hints["return"] is ReasoningPartDict def test_tool_call_part_to_dict_annotation(self): import typing + hints = typing.get_type_hints(llm.ToolCallPart.to_dict) assert hints["return"] is ToolCallPartDict def test_tool_result_part_to_dict_annotation(self): import typing + hints = typing.get_type_hints(llm.ToolResultPart.to_dict) assert hints["return"] is ToolResultPartDict def test_attachment_part_to_dict_annotation(self): import typing + hints = typing.get_type_hints(llm.AttachmentPart.to_dict) assert hints["return"] is AttachmentPartDict def test_message_to_dict_annotation(self): import typing + hints = typing.get_type_hints(llm.Message.to_dict) assert hints["return"] is MessageDict def test_message_from_dict_annotation(self): import typing + hints = typing.get_type_hints(llm.Message.from_dict) assert hints["d"] is MessageDict def test_response_to_dict_annotation(self): import typing + hints = typing.get_type_hints(llm.Response.to_dict) assert hints["return"] is ResponseDict From 92f9359ba3c5c99751fbe8a692308db6c78de585 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Apr 2026 21:24:50 -0700 Subject: [PATCH 035/258] Documentation improvements --- docs/python-api.md | 47 ++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/docs/python-api.md b/docs/python-api.md index 26549a48d..31e9c6103 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -530,13 +530,19 @@ If a response has been evaluated, `response.text()` will continue to return the ### Structured messages and streaming events -LLM has a structured view of a conversation that sits alongside the simple string API. Each turn is an `llm.Message` with a `role` (`"user"`, `"assistant"`, `"system"`, or `"tool"`) and a list of `Part` objects — `TextPart`, `ReasoningPart`, `ToolCallPart`, `ToolResultPart`, or `AttachmentPart`. You can pass explicit structured input via `messages=[...]`, observe typed events as the model streams, and inspect the assembled message after the response completes. +Many LLMs return structure that goes beyond a plain text response. LLM represents these using **messages** that consist of **parts**. + +A conversation consists of turns, where each turn is an `llm.Message` with a `role` (`"user"`, `"assistant"`, `"system"`, or `"tool"`) and a list of `Part` objects — `TextPart`, `ReasoningPart`, `ToolCallPart`, `ToolResultPart`, or `AttachmentPart`. + +You can pass structured prompt inputs via `messages=[...]`, iterate over typed events as the model streams, and inspect the assembled message after the response completes. + +Here's how to prompt a model with a list of messages instead of a plain text prompt: ```python import llm from llm import user, assistant, system -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.4-mini") response = model.prompt(messages=[ system("You are a helpful pirate."), @@ -547,9 +553,9 @@ response = model.prompt(messages=[ print(response.text()) ``` -The `user`, `assistant`, `system`, and `tool_message` helpers accept strings (wrapped as `TextPart`), `llm.Attachment` instances (wrapped as `AttachmentPart`), existing `Part` objects, and nested lists or tuples. +The `user()`, `assistant()`, and `system()` helpers accept strings (wrapped as `TextPart`) but can also accept `llm.Attachment` instances (wrapped as `AttachmentPart`) or more complex sequences of `Part` objects. -The simple `model.prompt("hi", system="Be brief.")` form is equivalent to `model.prompt(messages=[system("Be brief."), user("hi")])`. +Calling `model.prompt("hi", system="Be brief.")` is equivalent to `model.prompt(messages=[system("Be brief."), user("hi")])`. #### Streaming events as they arrive @@ -570,11 +576,11 @@ for event in response.stream_events(): Event types are `"text"`, `"reasoning"`, `"tool_call_name"`, `"tool_call_args"`, and `"tool_result"`. Each event carries a `part_index` that groups events into the same logical Part (all events at the same `part_index` assemble into one Part after the stream completes). For async models, use `async for event in response.astream_events()`. -Plain iteration (`for chunk in response`) continues to yield only text strings — reasoning and tool-call events are filtered out. +Iterating against the response object itself (`for chunk in response`) yields only text strings — reasoning and tool-call events are filtered out. #### Inspecting the finished response -After a response completes, `response.messages` gives you the assembled list of `Message` objects: +After a response completes, `response.messages` gives you the assembled list of `Message` objects returned by that response, excluding the messages from the original prompt: ```python response = model.prompt("What's 2+2?") @@ -584,31 +590,32 @@ for message in response.messages: print(type(part).__name__, part.to_dict()) ``` -#### Persisting a conversation yourself +#### Persisting a conversation + +A `Response` can round-trip through a plain Python dictionary via `response.to_dict()` and `llm.Response.from_dict(...)`. The dict captures the model id, the input messages that were sent, the assistant output, and any options — everything needed to continue the conversation later. -Messages and Parts round-trip through plain Python dicts via `to_dict()` / `from_dict()`, so your application can persist conversations to any JSON-capable store without touching SQLite: +Use `response.reply(...)` to continue from a rehydrated response: ```python import json +import llm -# Turn 1 +model = llm.get_model("gpt-5.4-mini") response = model.prompt("What's 2+2?") -response.text() +print(response.text()) -# Build a history payload from the user prompt + assistant reply. -history = [user("What's 2+2?").to_dict()] + [ - m.to_dict() for m in response.messages -] -payload = json.dumps(history) +payload = json.dumps(response.to_dict()) # ...save `payload` wherever you want... -# Later — re-inflate and continue. -rebuilt = [llm.Message.from_dict(d) for d in json.loads(payload)] -response = model.prompt(messages=rebuilt + [user("And 3+3?")]) -print(response.text()) +# Later — rehydrate and continue. +rebuilt = llm.Response.from_dict(json.loads(payload)) +followup = rebuilt.reply("Add 3 to that") +print(followup.text()) ``` -`AttachmentPart` bytes are base64-encoded in the dict form, so full multi-modal conversations round-trip faithfully. +`AttachmentPart` bytes are base64-encoded in the dict form, so multi-modal conversations round-trip faithfully via JSON too. + +Individual `Message` and `Part` objects also support `to_dict()` / `from_dict()` if you need to manipulate turns directly — for example, to edit, filter, or splice messages before passing them back via `model.prompt(messages=[...])`. (python-api-async)= From afc41b0c4753b5d981fa27c7f8074ba7a5be2c05 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Apr 2026 21:37:04 -0700 Subject: [PATCH 036/258] mypy fixes --- llm/default_plugins/openai_models.py | 6 ++-- llm/models.py | 36 ++++++++++++++-------- llm/parts.py | 45 +++++++++++++++------------- 3 files changed, 51 insertions(+), 36 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 804655329..abbb7d166 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -807,7 +807,7 @@ def execute( response: Response, conversation: Optional[Conversation] = None, key: Optional[str] = None, - ) -> Iterator[str]: + ) -> Iterator[Union[str, StreamEvent]]: if prompt.system and not self.allows_system_prompt: raise NotImplementedError("Model does not support system prompts") messages = self.build_messages(prompt, conversation) @@ -946,7 +946,7 @@ async def execute( response: AsyncResponse, conversation: Optional[AsyncConversation] = None, key: Optional[str] = None, - ) -> AsyncGenerator[str, None]: + ) -> AsyncGenerator[Union[str, StreamEvent], None]: if prompt.system and not self.allows_system_prompt: raise NotImplementedError("Model does not support system prompts") messages = self.build_messages(prompt, conversation) @@ -1080,7 +1080,7 @@ def execute( response: Response, conversation: Optional[Conversation] = None, key: Optional[str] = None, - ) -> Iterator[str]: + ) -> Iterator[Union[str, StreamEvent]]: if prompt.system: raise NotImplementedError( "System prompts are not supported for OpenAI completion models" diff --git a/llm/models.py b/llm/models.py index 541ed5743..eb7d98f6d 100644 --- a/llm/models.py +++ b/llm/models.py @@ -12,6 +12,7 @@ import time from types import MethodType from typing import ( + TYPE_CHECKING, Any, AsyncGenerator, AsyncIterator, @@ -24,9 +25,13 @@ Optional, Set, Union, + cast, get_type_hints, ) from .serialization import ResponseDict + +if TYPE_CHECKING: + from .parts import StreamEvent from .utils import ( ensure_fragment, ensure_tool, @@ -884,6 +889,11 @@ def __init__( if self.prompt.tools and not self.model.supports_tools: raise ValueError(f"{self.model} does not support tools") + @property + def messages(self) -> List[Any]: + "Overridden by Response / AsyncResponse — declared here for type checkers." + raise NotImplementedError + def _process_chunk(self, chunk): """Normalize a chunk from execute() into a StreamEvent and return the text str (or None) that __iter__ should yield. @@ -934,12 +944,12 @@ def _build_parts(self) -> List[Any]: # _tool_calls so response.messages isn't empty after # from_row, and Conversation.prompt-built chains include # the assistant turn on follow-up calls. - parts: List[Any] = [] + fallback_parts: List[Any] = [] text = "".join(self._chunks) if text: - parts.append(TextPart(text=text)) + fallback_parts.append(TextPart(text=text)) for tc in self._tool_calls: - parts.append( + fallback_parts.append( ToolCallPart( name=tc.name, arguments=tc.arguments or {}, @@ -948,7 +958,7 @@ def _build_parts(self) -> List[Any]: ) reasoning_token_count = getattr(self, "_reasoning_token_count", 0) if reasoning_token_count: - parts.insert( + fallback_parts.insert( 0, ReasoningPart( text="", @@ -956,7 +966,7 @@ def _build_parts(self) -> List[Any]: token_count=reasoning_token_count, ), ) - return parts + return fallback_parts def family(t: str) -> str: if t in ("tool_call_name", "tool_call_args"): @@ -1438,11 +1448,11 @@ def _response_to_dict(response: "_BaseResponse") -> ResponseDict: payload["usage"] = usage if response._start_utcnow is not None: payload["datetime_utc"] = response._start_utcnow.isoformat() - return payload + return cast(ResponseDict, payload) def _response_from_dict( - data: Dict[str, Any], + data: ResponseDict, cls, *, model=None, @@ -1561,7 +1571,7 @@ def from_dict( ``model`` overrides the stored model id (useful for continuing on a different model). """ - return _response_from_dict(data, cls, model=model, async_=False) + return cast("Response", _response_from_dict(data, cls, model=model, async_=False)) def on_done(self, callback): "Register a callback to be called when the response is complete." @@ -1862,7 +1872,7 @@ def from_dict( model: Optional["AsyncModel"] = None, ) -> "AsyncResponse": """Async counterpart of Response.from_dict().""" - return _response_from_dict(data, cls, model=model, async_=True) + return cast("AsyncResponse", _response_from_dict(data, cls, model=model, async_=True)) @classmethod def from_row(cls, db, row, _async=False): @@ -2683,7 +2693,7 @@ def execute( stream: bool, response: Response, conversation: Optional[Conversation], - ) -> Iterator[str]: + ) -> Iterator[Union[str, "StreamEvent"]]: pass @@ -2696,7 +2706,7 @@ def execute( response: Response, conversation: Optional[Conversation], key: Optional[str], - ) -> Iterator[str]: + ) -> Iterator[Union[str, "StreamEvent"]]: pass @@ -2796,7 +2806,7 @@ async def execute( stream: bool, response: AsyncResponse, conversation: Optional[AsyncConversation], - ) -> AsyncGenerator[str, None]: + ) -> AsyncGenerator[Union[str, "StreamEvent"], None]: if False: # Ensure it's a generator type yield "" pass @@ -2811,7 +2821,7 @@ async def execute( response: AsyncResponse, conversation: Optional[AsyncConversation], key: Optional[str], - ) -> AsyncGenerator[str, None]: + ) -> AsyncGenerator[Union[str, "StreamEvent"], None]: if False: # Ensure it's a generator type yield "" pass diff --git a/llm/parts.py b/llm/parts.py index 3c12f1ef1..1e23db7ed 100644 --- a/llm/parts.py +++ b/llm/parts.py @@ -42,14 +42,15 @@ def _attachment_to_dict(att: Attachment) -> AttachmentDict: def _attachment_from_dict(d: AttachmentDict) -> Attachment: - content = d.get("content") - if isinstance(content, str): - content = base64.b64decode(content) + raw_content = d.get("content") + content_bytes: Optional[bytes] = None + if isinstance(raw_content, str): + content_bytes = base64.b64decode(raw_content) return Attachment( type=d.get("type"), path=d.get("path"), url=d.get("url"), - content=content, + content=content_bytes, ) @@ -62,42 +63,46 @@ def to_dict(self) -> PartDict: @staticmethod def from_dict(d: PartDict) -> "Part": - type_ = d.get("type") - pm = d.get("provider_metadata") - if type_ == "text": - return TextPart(text=d.get("text", ""), provider_metadata=pm) - if type_ == "reasoning": + if d["type"] == "text": + return TextPart( + text=d["text"], + provider_metadata=d.get("provider_metadata"), + ) + if d["type"] == "reasoning": return ReasoningPart( - text=d.get("text", ""), + text=d["text"], redacted=d.get("redacted", False), token_count=d.get("token_count"), - provider_metadata=pm, + provider_metadata=d.get("provider_metadata"), ) - if type_ == "tool_call": + if d["type"] == "tool_call": return ToolCallPart( name=d["name"], - arguments=d.get("arguments", {}), + arguments=d["arguments"], tool_call_id=d.get("tool_call_id"), server_executed=d.get("server_executed", False), - provider_metadata=pm, + provider_metadata=d.get("provider_metadata"), ) - if type_ == "tool_result": + if d["type"] == "tool_result": return ToolResultPart( name=d["name"], - output=d.get("output", ""), + output=d["output"], tool_call_id=d.get("tool_call_id"), server_executed=d.get("server_executed", False), exception=d.get("exception"), attachments=[ _attachment_from_dict(a) for a in d.get("attachments", []) ], - provider_metadata=pm, + provider_metadata=d.get("provider_metadata"), ) - if type_ == "attachment": + if d["type"] == "attachment": att_dict = d.get("attachment") attachment = _attachment_from_dict(att_dict) if att_dict else None - return AttachmentPart(attachment=attachment, provider_metadata=pm) - raise ValueError(f"Unknown part type: {type_!r}") + return AttachmentPart( + attachment=attachment, + provider_metadata=d.get("provider_metadata"), + ) + raise ValueError(f"Unknown part type: {d['type']!r}") @dataclass From 8c48dccc942a537fded63b9ade49633a301bc506 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Apr 2026 21:43:42 -0700 Subject: [PATCH 037/258] docs: advanced-model-plugins.md covers StreamEvent / prompt.messages / provider_metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Substantially expanded docs/plugins/advanced-model-plugins.md with the plugin-author guide to the new machinery. Distilled from the actual llm-anthropic and llm-gemini implementations so plugin authors have a recipe that mirrors what real providers do. New / updated sections (doc grew 317 → 663 lines): - "Attachments from previous conversations" trimmed to a pointer at prompt.messages — the old pattern of walking conversation.responses is replaced by the canonical chain view. - "Structured messages and streaming events" - StreamEvent contract, backward compat for plain-str plugins - Full field reference (type / chunk / part_index / tool_call_id / provider_metadata / server_executed / tool_name) - part_index allocation rules with two worked examples: block-keyed (Anthropic-style content_block_start events) and kind-tracking (Gemini-style chunk-per-part) - Reasoning: streamed text + opaque _reasoning_token_count, with the OpenAI-specific gotcha about reading reasoning_tokens BEFORE set_usage mutates the dict - Tool calls — tool_call_name + tool_call_args pattern, reminder that response.add_tool_call() is separately required for chain-execution - Server-side tools — server_executed=True on events, raw payload in provider_metadata for round-trip, post-stream emission for providers that don't stream tool-result contents - Opaque provider_metadata — Anthropic signature, Gemini thoughtSignature, OpenAI encrypted_content — with namespacing guidance - Non-streaming path — one event per content block - "Consuming prompt.messages in build_messages" - The invariant: prompt.messages is always the full chain; don't walk conversation.responses (would double-emit) - Worked build_messages example that dispatches per Part subtype - Role mapping across OpenAI / Anthropic / Gemini conventions - Role-alternation merging - "Restoring opaque metadata on subsequent requests" - How to read provider_metadata off prior-turn Parts and fold the signatures back into the outgoing request body 670 tests still green. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/plugins/advanced-model-plugins.md | 384 +++++++++++++++++++++++-- 1 file changed, 367 insertions(+), 17 deletions(-) diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index d65edf629..e231c1e3d 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -246,30 +246,380 @@ As you can see, it uses `attachment.url` if that is available and otherwise fall ### Attachments from previous conversations -Models that implement the ability to continue a conversation can reconstruct the previous message JSON using the `response.attachments` attribute. +Conversation history — including attachments from prior turns — is available on the canonical `prompt.messages` list. See the [next section](#structured-messages-streaming) for how that works. -Here's how the OpenAI plugin does that: +(structured-messages-streaming)= + +## Structured messages and streaming events + +Modern plugins use a richer contract than "yield strings": + +1. **`execute()` yields `StreamEvent` objects** (or plain `str`, still supported) so text, reasoning (thinking tokens), tool calls, and server-side tool results each surface as their own event type. The framework assembles these into typed `Part` objects. +2. **`build_messages` (or equivalent) reads `prompt.messages`** — a `list[llm.Message]` that is the complete input chain for this turn. This replaces the older pattern of walking `conversation.responses` and reading `prompt.prompt` / `prompt.system`. +3. **Opaque provider tokens round-trip via `provider_metadata`** — Anthropic thinking signatures, Gemini thought signatures, OpenAI Responses API encrypted reasoning blobs. Plugins stash whatever the API returns, echo it back on the next request. + +**Backward compatibility is guaranteed.** A plugin that still yields plain `str` from `execute()` works unchanged — each string is wrapped as a `StreamEvent(type="text", chunk=..., part_index=0)` internally. + +### Yielding StreamEvent from execute() + +```python +from llm.parts import StreamEvent + +def execute(self, prompt, stream, response, conversation, key=None): + messages = self.build_messages(prompt, conversation) + ... + + for chunk in provider_sdk.stream(...): + if chunk.type == "text": + yield StreamEvent(type="text", chunk=chunk.text, part_index=0) + elif chunk.type == "thinking": + yield StreamEvent(type="reasoning", chunk=chunk.text, part_index=0) +``` + +A `StreamEvent` has five frequently-used fields: + +- **`type`** — one of `"text"`, `"reasoning"`, `"tool_call_name"`, `"tool_call_args"`, `"tool_result"`. +- **`chunk`** — the text fragment. For tool calls this is the tool name (for `tool_call_name`) or a partial JSON string (for `tool_call_args`). +- **`part_index`** — a monotonically allocated integer identifying which `Part` this event contributes to. All events sharing a `part_index` must belong to the same family; events at the same index concatenate into one Part. +- **`tool_call_id`** — the provider's id for the tool call, set on `tool_call_name` / `tool_call_args` / `tool_result` events. +- **`provider_metadata`** — an optional `dict[str, dict]` namespaced by provider name. Carries opaque data (signatures, encrypted blobs) that must be echoed back on future requests. + +Two additional fields exist for special cases: + +- **`server_executed: bool`** — set `True` for server-side tool calls (for example, Anthropic web search) and their results. The model ran the tool internally. +- **`tool_name`** — set on `tool_result` events to identify which tool this result came from. + +### Allocating `part_index` + +`part_index` groups events into Parts. Rules: + +- **Same `part_index` + same family** → events are appended into one Part (text concatenates; tool-call args accumulate into the final JSON). +- **Same `part_index` + different family** → the framework raises `ValueError`. That's a plugin bug — allocate a new index when a new content block begins. +- **Tool calls span two event types**: `tool_call_name` and `tool_call_args` at the *same* `part_index` combine into one `ToolCallPart`. The name arrives first; the args stream in as partial JSON and are parsed when the part finalizes. + +A typical allocation scheme: + +``` +part_index=0 reasoning chunks +part_index=1 text chunks +part_index=2 first tool_call (name + streaming args) +part_index=3 second tool_call (parallel) +``` + +For providers that emit discrete content blocks (like Anthropic's `content_block_start` / `content_block_delta` events), a natural implementation is a dict keyed by block index: + +```python +state = {"blocks": {}, "next_part_index": 0} + +# On content_block_start: +idx = event.index +pi = state["next_part_index"] +state["next_part_index"] += 1 +state["blocks"][idx] = {"kind": block.type, "part_index": pi} + +# On content_block_delta: +info = state["blocks"][event.index] +pi = info["part_index"] +yield StreamEvent(type="text", chunk=delta.text, part_index=pi) +``` + +For providers that emit discrete parts per streamed chunk without start/stop markers (like Gemini), track the current block kind and advance the index on kind changes: ```python -for prev_response in conversation.responses: - if prev_response.attachments: - attachment_message = [] - if prev_response.prompt.prompt: - attachment_message.append( - {"type": "text", "text": prev_response.prompt.prompt} +state = {"index": 0, "kind": None} + +def allocate_for_kind(state, new_kind): + if state["kind"] == new_kind: + return state["index"] # concat + if state["kind"] is not None: + state["index"] += 1 # advance past previous block + state["kind"] = new_kind + return state["index"] +``` + +### Reasoning tokens + +Two modes are supported: + +**Streamed reasoning text** (Anthropic extended thinking, Gemini with `includeThoughts: true`): + +```python +yield StreamEvent(type="reasoning", chunk=thinking_chunk, part_index=0) +``` + +Text events and reasoning events at different indexes produce distinct `TextPart` and `ReasoningPart` entries in `response.messages`. + +**Opaque reasoning token count** (OpenAI o-series, Gemini without `includeThoughts`): + +The provider reports only a count — no reasoning text. Record the count on the Response object and the framework will prepend a redacted `ReasoningPart`: + +```python +# Anywhere before set_usage runs (usually at the end of execute): +if reasoning_tokens > 0: + response._reasoning_token_count = reasoning_tokens +``` + +For OpenAI this count lives in `usage.completion_tokens_details.reasoning_tokens`; read it **before** calling `self.set_usage()` — `set_usage()` mutates the usage dict via `pop()` and simplifies out zero-valued entries. + +### Tool calls + +Each tool call emits two event types at the same `part_index`: + +```python +yield StreamEvent( + type="tool_call_name", + chunk=tool_name, + part_index=tc_part_index, + tool_call_id=tool_call_id, +) +# then, as the provider streams JSON args: +yield StreamEvent( + type="tool_call_args", + chunk=partial_json_fragment, + part_index=tc_part_index, + tool_call_id=tool_call_id, +) +``` + +Some providers (Gemini) emit the complete tool call in one chunk — fine; emit both events back-to-back with the full name and full JSON. + +For client-side tool calls — tools that LLM should execute locally in a chain — **also call `response.add_tool_call()`**. The chain-execution path (`response.tool_calls()` → `execute_tool_calls()`) reads from the explicitly-added list, not from the StreamEvent buffer. Your code should do both: + +```python +response.add_tool_call( + llm.ToolCall( + tool_call_id=tool_id, + name=tool_name, + arguments=parsed_args, + ) +) +``` + +### Server-side tool calls + +For tools the API executes internally, set `server_executed=True` on the events. Anthropic web search is a good concrete example: the API returns a `server_tool_use` block for the search request, followed by a `web_search_tool_result` block containing the result payload. + +```python +yield StreamEvent( + type="tool_call_name", + chunk="web_search", + part_index=tc_pi, + tool_call_id=tool_id, + server_executed=True, +) +yield StreamEvent( + type="tool_call_args", + chunk=json.dumps(query_args), + part_index=tc_pi, + tool_call_id=tool_id, + server_executed=True, +) +``` + +The tool *result* (for example, the search hits) is also emitted as an event: + +```python +yield StreamEvent( + type="tool_result", + chunk=human_readable_summary, + part_index=tr_pi, + tool_call_id=tool_id, + server_executed=True, + tool_name="web_search", + provider_metadata={"myprovider": {"raw_content": full_payload}}, +) +``` + +For providers that don't stream server-tool-result contents (Anthropic's `web_search_tool_result` blocks only arrive in the final message), do the emission as a post-stream step — after the main iteration loop completes, inspect the final message and emit tool_result events for any server-side results. + +Do **not** call `response.add_tool_call()` for server-side tool calls unless you intentionally want LLM to run a separate local tool too. The provider has already executed these calls; represent them as `StreamEvent`s so they are preserved in `response.messages` and can be replayed in future turns. + +### Opaque provider metadata + +Some providers require you to echo back opaque fields on the next request for multi-turn continuity to work: + +- **Anthropic** — `signature` on each thinking block; `encrypted_content` inside web_search_tool_result items. +- **Google Gemini** — `thoughtSignature` on `functionCall` parts when thinking is active. +- **OpenAI Responses API** — `encrypted_content` on reasoning items in stateless mode. + +These values are attached to a `StreamEvent` via its `provider_metadata` field. The framework merges metadata across events at the same `part_index` (last non-None wins per top-level key) and persists it on the finalized Part. + +Namespace under your provider's name so transcripts that mix providers don't collide: + +```python +# Anthropic signature arrives at the end of a thinking block. +yield StreamEvent( + type="reasoning", + chunk="", + part_index=reasoning_pi, + provider_metadata={"anthropic": {"signature": sig}}, +) +``` + +```python +# Gemini attaches thoughtSignature to a functionCall part. +yield StreamEvent( + type="tool_call_name", + chunk=name, + part_index=tc_pi, + tool_call_id=tc_id, + provider_metadata={"gemini": {"thoughtSignature": sig}}, +) +``` + +Treat other providers' entries as opaque; don't parse them. The framework round-trips the value verbatim via JSON, so use JSON-safe primitives (string, int, bool, dict, list) — avoid custom classes or bytes (base64-encode bytes if you need them). + +### Non-streaming path + +When `stream=False` (or the provider returns a complete message at once), emit one event per content block: + +```python +else: + completion = client.messages.create(**kwargs) + response.response_json = completion.model_dump() + pi = 0 + for block in completion.content: + if block.type == "thinking": + yield StreamEvent( + type="reasoning", + chunk=block.thinking, + part_index=pi, + provider_metadata={"anthropic": {"signature": block.signature}}, + ) + pi += 1 + elif block.type == "text": + yield StreamEvent(type="text", chunk=block.text, part_index=pi) + pi += 1 + elif block.type == "tool_use": + yield StreamEvent( + type="tool_call_name", + chunk=block.name, + part_index=pi, + tool_call_id=block.id, ) - for attachment in prev_response.attachments: - attachment_message.append(_attachment(attachment)) - messages.append({"role": "user", "content": attachment_message}) + yield StreamEvent( + type="tool_call_args", + chunk=json.dumps(block.input), + part_index=pi, + tool_call_id=block.id, + ) + pi += 1 +``` + +## Consuming prompt.messages in build_messages + +`prompt.messages` is an `list[llm.Message]` that is always **the complete input chain for this turn** — whether the caller supplied it explicitly via `model.prompt(messages=[...])`, or it was synthesized from legacy kwargs (`prompt=`, `system=`, `attachments=`, `tool_results=`), or it was pre-built by a `Conversation` or by `response.reply()`. + +**Do not also walk `conversation.responses`.** Under the invariant, history is already baked into `prompt.messages`; walking the conversation would double-emit. + +A plugin's `build_messages` (or equivalent) iterates `prompt.messages` and dispatches per `Part` subtype: + +```python +from llm.parts import ( + TextPart, + ReasoningPart, + ToolCallPart, + ToolResultPart, + AttachmentPart, +) + +def build_messages(self, prompt, conversation): + messages = [] + for msg in prompt.messages: + if msg.role == "system": + # Some APIs put system on a separate kwarg (Anthropic, Gemini). + # OpenAI-style APIs emit it as a message; handle accordingly. + continue + self._append_message(messages, msg) + return messages + +def _append_message(self, out, msg): + # Map llm's role to the provider's (assistant→model for Gemini, + # tool→user for Anthropic/Gemini tool_result convention, etc.) + role = self._provider_role(msg.role) + parts = [] + for part in msg.parts: + if isinstance(part, TextPart): + parts.append({"type": "text", "text": part.text}) + elif isinstance(part, ReasoningPart): + # Skip redacted reasoning (no content to echo back). + if part.redacted or not part.text: + continue + block = {"type": "thinking", "thinking": part.text} + # Restore the signature from provider_metadata. + sig = (part.provider_metadata or {}).get("anthropic", {}).get("signature") + if sig: + block["signature"] = sig + parts.append(block) + elif isinstance(part, ToolCallPart): + parts.append({ + "type": "tool_use", + "id": part.tool_call_id, + "name": part.name, + "input": part.arguments, + }) + elif isinstance(part, ToolResultPart): + parts.append({ + "type": "tool_result", + "tool_use_id": part.tool_call_id, + "content": part.output, + }) + elif isinstance(part, AttachmentPart) and part.attachment: + parts.append(self._attachment_block(part.attachment)) + # Merge with the previous message if roles match (some providers + # require strict alternation between user and assistant). + if out and out[-1]["role"] == role: + out[-1]["content"].extend(parts) else: - messages.append( - {"role": "user", "content": prev_response.prompt.prompt} - ) - messages.append({"role": "assistant", "content": prev_response.text_or_raise()}) + out.append({"role": role, "content": parts}) +``` + +### Role mapping + +LLM uses four roles: `"user"`, `"assistant"`, `"system"`, `"tool"`. Providers differ: + +- **OpenAI Chat Completions** — carries system in the messages array. `"tool"` → `{"role": "tool", "tool_call_id": ..., "content": ...}` per result. +- **Anthropic Messages** — system on a separate `system=` kwarg. `"tool"` → user-role message with `tool_result` blocks. `"assistant"` unchanged. +- **Gemini Generate Content** — system on `systemInstruction`. `"assistant"` → `"model"`. `"tool"` → user-role with `function_response` parts. + +For adapters that need system separately: filter `msg.role == "system"` out of the messages loop and read the current-turn system from `prompt.system` (the synthesized string of `prompt._system` + any system_fragments). The `prompt.system` attribute remains populated by Conversation.prompt for backward compatibility. + +### Role-alternation merging + +Several providers require strict alternation between user and assistant (or equivalent) messages. When two consecutive `llm.Message` values map to the same provider-side role, merge their parts into one provider message: + +```python +if out and out[-1]["role"] == role: + out[-1]["content"].extend(parts) +else: + out.append({"role": role, "content": parts}) +``` + +This is especially relevant for `tool` + `user` — both typically map to a `user` turn for Anthropic/Gemini. + +## Restoring opaque metadata on subsequent requests + +When a conversation continues, your `build_messages` walks prior-turn Parts via `prompt.messages`. Each Part's `provider_metadata` is a `dict[str, dict]` keyed by provider name — extract your namespace and fold the fields back into the outgoing request body: + +```python +if isinstance(part, ReasoningPart): + block = {"type": "thinking", "thinking": part.text} + pm = (part.provider_metadata or {}).get("anthropic", {}) + if "signature" in pm: + block["signature"] = pm["signature"] + parts.append(block) + +if isinstance(part, ToolCallPart): + fc_part = {"function_call": {"name": part.name, "args": part.arguments}} + pm = (part.provider_metadata or {}).get("gemini", {}) + if "thoughtSignature" in pm: + # Gemini expects thoughtSignature beside function_call, + # not nested inside it. + fc_part["thoughtSignature"] = pm["thoughtSignature"] + parts.append(fc_part) ``` -The `response.text_or_raise()` method used there will return the text from the response or raise a `ValueError` exception if the response is an `AsyncResponse` instance that has not yet been fully resolved. -This is a slightly weird hack to work around the common need to share logic for building up the `messages` list across both sync and async models. +If the key is missing (an older transcript that pre-dates your plugin's support), fall through — don't fail. Treat other providers' entries as opaque; don't parse them. (advanced-model-plugins-usage)= From 00e428896f2db5fb9ab2aa90cc20fcf60e3f4d8b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 22 Apr 2026 04:54:40 +0000 Subject: [PATCH 038/258] Ran cog --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 526b926bb..d3da2444e 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,9 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Supporting schemas](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-schemas) * [Supporting tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-tools) * [Attachments for multi-modal models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#attachments-for-multi-modal-models) + * [Structured messages and streaming events](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#structured-messages-and-streaming-events) + * [Consuming prompt.messages in build_messages](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#consuming-prompt-messages-in-build-messages) + * [Restoring opaque metadata on subsequent requests](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#restoring-opaque-metadata-on-subsequent-requests) * [Tracking token usage](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-token-usage) * [Tracking resolved model names](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-resolved-model-names) * [LLM_RAISE_ERRORS](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#llm-raise-errors) From d11b9a01c525ac5d998880d8244216537912210b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 22 Apr 2026 08:34:12 -0700 Subject: [PATCH 039/258] More documentation tweaks --- docs/plugins/advanced-model-plugins.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index e231c1e3d..8e65641ca 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -252,11 +252,11 @@ Conversation history — including attachments from prior turns — is available ## Structured messages and streaming events -Modern plugins use a richer contract than "yield strings": +The 0.31 alpha introduced a richer contract for plugins than "yield strings": 1. **`execute()` yields `StreamEvent` objects** (or plain `str`, still supported) so text, reasoning (thinking tokens), tool calls, and server-side tool results each surface as their own event type. The framework assembles these into typed `Part` objects. -2. **`build_messages` (or equivalent) reads `prompt.messages`** — a `list[llm.Message]` that is the complete input chain for this turn. This replaces the older pattern of walking `conversation.responses` and reading `prompt.prompt` / `prompt.system`. -3. **Opaque provider tokens round-trip via `provider_metadata`** — Anthropic thinking signatures, Gemini thought signatures, OpenAI Responses API encrypted reasoning blobs. Plugins stash whatever the API returns, echo it back on the next request. +2. **`build_messages` (or equivalent) reads `prompt.messages`** — a `list[llm.Message]` that is the complete input chain for this turn. +3. **Opaque provider tokens round-trip via `provider_metadata`** — Anthropic thinking signatures, Gemini thought signatures, OpenAI Responses API encrypted reasoning blobs. Plugins stash whatever the API returns, then echo it back on the next request. **Backward compatibility is guaranteed.** A plugin that still yields plain `str` from `execute()` works unchanged — each string is wrapped as a `StreamEvent(type="text", chunk=..., part_index=0)` internally. From 211e678e0730d0489426c61f502d414d782db906 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 22 Apr 2026 08:43:33 -0700 Subject: [PATCH 040/258] Cleaned up tests and comments Removed all mentions of 'phase' --- llm/default_plugins/openai_models.py | 7 --- llm/models.py | 10 ++-- tests/test_async_parity.py | 2 +- tests/test_openai_messages.py | 19 ------- tests/test_parts.py | 74 +++------------------------- 5 files changed, 15 insertions(+), 97 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index abbb7d166..db31d7646 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -712,13 +712,6 @@ def _append_llm_message(self, out, message, current_system): return current_system def build_messages(self, prompt, conversation): - """Translate prompt.messages into OpenAI's wire format. - - Under the Phase 7 invariant, ``prompt.messages`` is the full - chain for this turn — Conversation.prompt and response.reply - pre-bake the history into it. The ``conversation`` parameter - is unused and retained only for the plugin API contract. - """ messages: List[Dict[str, Any]] = [] current_system: Optional[str] = None for msg in prompt.messages: diff --git a/llm/models.py b/llm/models.py index eb7d98f6d..654a4565e 100644 --- a/llm/models.py +++ b/llm/models.py @@ -1571,7 +1571,9 @@ def from_dict( ``model`` overrides the stored model id (useful for continuing on a different model). """ - return cast("Response", _response_from_dict(data, cls, model=model, async_=False)) + return cast( + "Response", _response_from_dict(data, cls, model=model, async_=False) + ) def on_done(self, callback): "Register a callback to be called when the response is complete." @@ -1804,7 +1806,7 @@ def messages(self) -> List[Any]: Almost always a single assistant Message; multiple messages are possible for providers that emit multi-message responses during - server-side tool execution (not in this phase's scope). + server-side tool execution. Responses rehydrated via ``Response.from_dict`` short-circuit and return the stored messages directly. @@ -1872,7 +1874,9 @@ def from_dict( model: Optional["AsyncModel"] = None, ) -> "AsyncResponse": """Async counterpart of Response.from_dict().""" - return cast("AsyncResponse", _response_from_dict(data, cls, model=model, async_=True)) + return cast( + "AsyncResponse", _response_from_dict(data, cls, model=model, async_=True) + ) @classmethod def from_row(cls, db, row, _async=False): diff --git a/tests/test_async_parity.py b/tests/test_async_parity.py index 5c2479a6f..f6acfc306 100644 --- a/tests/test_async_parity.py +++ b/tests/test_async_parity.py @@ -1,4 +1,4 @@ -"""Async parity: every sync API added in Phases 1-7 must work the same +"""Async parity: every sync API must work the same way on AsyncResponse and AsyncConversation. Uses the llm-echo plugin (sync ``Echo`` + async ``EchoAsync``) so both diff --git a/tests/test_openai_messages.py b/tests/test_openai_messages.py index 459633a1e..534ab0895 100644 --- a/tests/test_openai_messages.py +++ b/tests/test_openai_messages.py @@ -1,13 +1,3 @@ -"""Tests for the OpenAI built-in plugin's messages= path. - -Phase 4a covers build_messages reading prompt.messages (instead of the -legacy prompt.prompt / prompt.system / prompt.attachments fields), which -lets users pass structured message history via model.prompt(messages=[...]). - -Phase 4b covers execute() yielding StreamEvent objects instead of plain -str — including tool_call_name + tool_call_args event streams. -""" - import json import pytest @@ -316,9 +306,6 @@ def test_system_change_emitted(self, chat_model): class TestBuildMessagesConversationHistory: def test_prior_turn_text_plus_current_user(self, chat_model): - """With the Phase 7 invariant, prompt.messages for a follow-up - turn already contains the full chain — the adapter reads only - from it, not from conversation.responses.""" new_prompt = Prompt( None, model=chat_model, @@ -338,9 +325,6 @@ def test_prior_turn_text_plus_current_user(self, chat_model): def test_no_double_emission_from_conversation_prompt_flow( self, chat_model, httpx_mock ): - """Phase 7 invariant: prompt.messages for a conversation's - follow-up turn is the full chain. The adapter must not ALSO - walk conversation.responses, or the wire body doubles up.""" # Two staged responses so conv.prompt twice can complete. httpx_mock.add_response( method="POST", @@ -399,9 +383,6 @@ def test_no_double_emission_from_conversation_prompt_flow( ] -# -- Phase 4b: execute() yields StreamEvents --------------------------- - - class TestStreamingExecuteYieldsStreamEvents: def test_text_stream_yields_text_events(self, httpx_mock): httpx_mock.add_response( diff --git a/tests/test_parts.py b/tests/test_parts.py index 779f221a7..c37da7fbd 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -1,37 +1,7 @@ -"""Tests for Part, Message, StreamEvent and the constructor helpers. - -Phase 1 covers the in-memory value types and JSON round-trip only. -No Response / streaming / plugin integration yet. -""" - import json import pytest - import llm -# -- Exports ------------------------------------------------------------ - - -class TestExports: - def test_llm_exports_part_types(self): - assert llm.Part is not None - assert llm.TextPart is not None - assert llm.ReasoningPart is not None - assert llm.ToolCallPart is not None - assert llm.ToolResultPart is not None - assert llm.AttachmentPart is not None - assert llm.Message is not None - assert llm.StreamEvent is not None - - def test_llm_exports_constructor_helpers(self): - assert callable(llm.user) - assert callable(llm.assistant) - assert callable(llm.system) - assert callable(llm.tool_message) - - -# -- Part subclasses ---------------------------------------------------- - class TestTextPart: def test_roundtrip(self): @@ -167,9 +137,6 @@ def test_tool_call_part_has_no_role_attribute(self): ) -# -- Message ------------------------------------------------------------ - - class TestMessage: def test_roundtrip_simple_user_message(self): m = llm.Message(role="user", parts=[llm.TextPart(text="hi")]) @@ -217,9 +184,6 @@ def test_none_and_empty_provider_metadata_equivalent(self): assert m_none.to_dict() == m_empty.to_dict() -# -- Constructor helpers ----------------------------------------------- - - class TestHelpers: def test_user_with_string(self): m = llm.user("hi") @@ -273,9 +237,6 @@ def test_helper_with_provider_metadata(self): assert m.provider_metadata == {"openai": {"id": "x"}} -# -- StreamEvent (type only, no Response integration yet) -------------- - - class TestStreamEvent: def test_dataclass_defaults(self): ev = llm.StreamEvent(type="text", chunk="hi", part_index=0) @@ -306,16 +267,9 @@ def test_all_fields_accepted(self): assert ev.message_index == 1 -# -- Phase 2: Response streaming scaffolding ---------------------------- -# # Backward compat for plain-str plugins: iterating a Response still # yields text strings, response.text() still works, self._chunks is # still populated. -# -# New capabilities: -# - response.stream_events() / response.astream_events() -# - response.messages -# - _BaseResponse._build_parts() (internal, tested via .messages) class TestPlainStrPluginCompat: @@ -523,8 +477,7 @@ def execute(self, prompt, stream, response, conversation): class TestStreamEventsLiveDuringStreaming: - """Client code sees events arrive before the response is done — - this is the primary user-facing goal of this phase.""" + """Client code sees events arrive before the response is done""" def test_events_arrive_before_done(self, mock_model): events = [ @@ -607,9 +560,6 @@ async def test_async_messages_after_await(self, async_mock_model): ] -# -- Phase 3: messages= parameter and Prompt.messages synthesis -------- - - class TestPromptMessagesSynthesis: """Prompt.messages constructs a Message list from legacy inputs when messages= wasn't passed explicitly.""" @@ -739,8 +689,6 @@ async def test_async_conversation_prompt_accepts_messages(self, async_mock_model assert response.prompt.messages == [llm.user("q")] -# -- Phase 7.1: Conversation passes full chain via messages= ---------- -# # Invariant: response.prompt.messages == exactly what the model was # sent for this turn, regardless of whether the caller used # model.prompt(messages=[...]), conversation.prompt("text"), or @@ -850,9 +798,6 @@ async def test_async_conversation_full_chain(self, async_mock_model): ] -# -- Regression: rehydrated-from-SQLite response.messages survives ---- - - class TestSqliteRehydrateMessages: """After Response.from_row, response.messages must still yield the assistant turn as a TextPart (+ any tool calls). Otherwise @@ -914,9 +859,6 @@ def test_llm_dash_c_chain_preserves_prior_assistant_turn( ] -# -- Phase 7.3: response.reply() -------------------------------------- - - class TestResponseReply: def test_reply_builds_next_turn_from_this_response(self, mock_model): mock_model.enqueue(["a1"]) @@ -994,7 +936,7 @@ async def test_async_reply(self, async_mock_model): ] -# -- chain() propagates system across tool-result turns -------------- +# chain() propagates system across tool-result turns class TestChainPropagatesSystem: @@ -1103,7 +1045,7 @@ def tick() -> str: assert second.prompt.system == "be brief" -# -- chain() accepts messages= (parity with prompt()) ----------------- +# chain() accepts messages= (parity with prompt()) class TestChainMessagesKwarg: @@ -1168,7 +1110,7 @@ async def test_async_model_chain_accepts_messages(self, async_mock_model): assert r1.prompt.messages == [llm.user("explicit")] -# -- Phase 7.2: Response.to_dict / Response.from_dict ------------------ +# Response.to_dict / Response.from_dict class TestResponseToDictFromDict: @@ -1240,9 +1182,8 @@ def test_to_dict_preserves_reasoning_and_signatures(self, mock_model): } def test_from_dict_reply_includes_prior_reasoning_in_chain(self, mock_model): - """The thing this entire refactor was about: a reply() after - from_dict() sends the thinking signature back to the model - for multi-turn extended thinking.""" + """a reply() after from_dict() sends the thinking signature + back to the model for multi-turn extended thinking.""" mock_model.enqueue( [ llm.StreamEvent( @@ -1291,7 +1232,6 @@ def test_from_dict_options_restored(self, mock_model): assert restored.prompt.options.max_tokens == 42 def test_message_from_dict_static_method_unchanged(self): - # Sanity: Message.from_dict / to_dict keep the Phase 1 contract. m = llm.assistant("hi") assert llm.Message.from_dict(m.to_dict()) == m @@ -1318,7 +1258,7 @@ async def test_async_chain_astream_events_yields(self, async_mock_model): assert [e.type for e in events] == ["text"] -# -- Phase 6: Client-side serialization round-trip --------------------- +# Client-side serialization round-trip # # A library user can persist a conversation by serializing response.messages # to JSON and later re-inflate it as messages=[...] on a follow-up prompt. From 5a92cdfc6ec8ded375d53e5bea0d02bca2b131e6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 22 Apr 2026 08:55:38 -0700 Subject: [PATCH 041/258] Improved some comments Had a different model review them for accuracy --- llm/default_plugins/openai_models.py | 12 +++++------- llm/models.py | 27 +++++++++++---------------- llm/serialization.py | 3 ++- tests/test_openai_messages.py | 2 +- tests/test_parts.py | 2 +- 5 files changed, 20 insertions(+), 26 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index db31d7646..17927ec7b 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -636,7 +636,7 @@ def _append_llm_message(self, out, message, current_system): dicts and append them to ``out``. Returns the (possibly updated) current_system value so the caller - can dedup consecutive identical system messages. + can avoid re-emitting an unchanged system prompt. """ from llm.parts import ( AttachmentPart, @@ -680,7 +680,7 @@ def _append_llm_message(self, out, message, current_system): out.extend(tool_results) return current_system - # System dedup — skip if we just emitted this exact system text. + # System dedup: skip if this text is already the active system prompt. if message.role == "system": text = "".join(text_bits) if text == current_system: @@ -816,10 +816,9 @@ def execute( ) chunks = [] tool_calls = {} - # part_index allocator. Text always uses 0. Each tool call - # at delta index i is assigned a part_index past any text - # that was seen, so _build_parts groups them correctly. - seen_text = False + # part_index allocator. Text events always use 0, so tool + # calls start at 1 and keep a stable index per OpenAI delta + # index. This keeps _build_parts from mixing families. tc_part_index = {} next_part_index = 1 for chunk in completion: @@ -859,7 +858,6 @@ def execute( if content: # Empty strings are noise (OpenAI's first chunk # with role=assistant has content=""). - seen_text = True yield StreamEvent(type="text", chunk=content, part_index=0) response.response_json = remove_dict_none_values(combine_chunks(chunks)) if tool_calls: diff --git a/llm/models.py b/llm/models.py index 654a4565e..f2da4ed68 100644 --- a/llm/models.py +++ b/llm/models.py @@ -507,16 +507,16 @@ def _build_full_chain( ) -> List[Any]: """Build the full message chain for the next turn. - Walks this conversation's responses to collect prior history, - then appends the new turn's content (explicit messages first, - or synthesized from prompt/attachments/tool_results). + Uses the last response's stored prompt chain to recover prior + history, then appends the new turn's content (explicit messages + first, or synthesized from prompt/attachments/tool_results). Returns the list that should be passed as ``messages=`` to the Prompt constructor so that ``response.prompt.messages`` equals exactly what the model sees. If ``explicit_messages`` is provided, the caller has opted out - of history walking — the list is used as-is. + of history reconstruction and the list is used as-is. """ from .parts import ( AttachmentPart, @@ -529,17 +529,12 @@ def _build_full_chain( return list(explicit_messages) chain: List[Any] = [] - for prev in self.responses: - # prev.prompt.messages already contains prev's full input - # chain under the new invariant, but for the FIRST hop into - # a conversation we defensively de-duplicate by only - # concatenating the last response's full chain (which - # transitively includes everything before it). - pass if self.responses: last = self.responses[-1] + # last.prompt.messages already contains the full input chain + # under the invariant, so use the last response only and then + # append that response's structured output. chain.extend(last.prompt.messages) - # Append that response's own output (structured messages). try: chain.extend(last.messages) except ValueError: @@ -1415,7 +1410,8 @@ def _response_to_dict(response: "_BaseResponse") -> ResponseDict: """Shared serializer for Response.to_dict / AsyncResponse.to_dict. The output is a JSON-safe dict — store it anywhere (file, Redis, - Postgres, HTTP body) and round-trip via Response.from_dict. + Postgres, HTTP body) and round-trip via Response.from_dict or + AsyncResponse.from_dict. """ options = { key: value @@ -1730,9 +1726,8 @@ def usage(self) -> Usage: ) def _iter_events(self): - """Drive self.model.execute() once. Yields every chunk it - produces, each already appended to self._stream_events by - _process_chunk as a side effect. + """Drive self.model.execute() once and yield each raw chunk it + produces. Callers normalize chunks through _process_chunk. """ if isinstance(self.model, Model): generator = self.model.execute( diff --git a/llm/serialization.py b/llm/serialization.py index 870542ae4..6ed401318 100644 --- a/llm/serialization.py +++ b/llm/serialization.py @@ -94,7 +94,8 @@ class ToolCallPartDict(TypedDict): arguments: Dict[str, Any] tool_call_id: NotRequired[str] # True for provider-executed calls (Anthropic web search, Gemini code - # execution). Client echoes the block back as-is on next turn. + # execution). Adapters use this to restore provider-side blocks on + # the next turn. server_executed: NotRequired[bool] provider_metadata: NotRequired[Dict[str, Any]] diff --git a/tests/test_openai_messages.py b/tests/test_openai_messages.py index 534ab0895..67734f84b 100644 --- a/tests/test_openai_messages.py +++ b/tests/test_openai_messages.py @@ -265,7 +265,7 @@ def test_attachments(self, chat_model): class TestBuildMessagesSystemDedup: """Explicit messages with repeated system messages dedupe - consecutive identical systems — OpenAI accepts one.""" + repeated unchanged systems; OpenAI accepts one.""" def test_same_system_not_repeated(self, chat_model): prompt = Prompt( diff --git a/tests/test_parts.py b/tests/test_parts.py index c37da7fbd..f0c572137 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -1077,7 +1077,7 @@ def test_chain_messages_is_authoritative_over_prompt_kwarg(self, mock_model): assert r1.prompt.messages == [llm.user("explicit")] def test_chain_with_messages_and_prior_conversation(self, mock_model): - """Explicit messages= on chain() replaces any history walking — + """Explicit messages= on chain() replaces history reconstruction; the chain starts from that exact list.""" mock_model.enqueue(["first"]) mock_model.enqueue(["second"]) From 38cf65adb117c42327fcff265814b915ebe22fac Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 22 Apr 2026 10:10:45 -0700 Subject: [PATCH 042/258] Remove unneccassry exception catch --- llm/models.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/llm/models.py b/llm/models.py index f2da4ed68..43bfcbc41 100644 --- a/llm/models.py +++ b/llm/models.py @@ -535,12 +535,7 @@ def _build_full_chain( # under the invariant, so use the last response only and then # append that response's structured output. chain.extend(last.prompt.messages) - try: - chain.extend(last.messages) - except ValueError: - # AsyncResponse not yet awaited — the caller shouldn't - # be constructing a next turn without awaiting first. - pass + chain.extend(last.messages) # Append the new turn's input if tool_results: From de63d8b69ea62dede300e15a165e75f8633734b8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 22 Apr 2026 10:13:39 -0700 Subject: [PATCH 043/258] Fixes for ruff --- llm/default_plugins/openai_models.py | 12 +++++++++++- llm/models.py | 1 - tests/test_async_parity.py | 1 - tests/test_serialization.py | 3 --- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 17927ec7b..2a406c2fb 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -26,7 +26,17 @@ from pydantic import field_validator, Field -from typing import AsyncGenerator, cast, List, Iterable, Iterator, Optional, Union +from typing import ( + Any, + AsyncGenerator, + cast, + Dict, + List, + Iterable, + Iterator, + Optional, + Union, +) import json import yaml diff --git a/llm/models.py b/llm/models.py index 43bfcbc41..ba5acc5f7 100644 --- a/llm/models.py +++ b/llm/models.py @@ -2290,7 +2290,6 @@ def _chain_for_tool_results(prior_response, tool_results, attachments) -> List[A from .parts import ( AttachmentPart, Message, - TextPart, ToolResultPart, ) diff --git a/tests/test_async_parity.py b/tests/test_async_parity.py index f6acfc306..08d571e8d 100644 --- a/tests/test_async_parity.py +++ b/tests/test_async_parity.py @@ -5,7 +5,6 @@ paths exercise real registered models with identical behaviour. """ -import asyncio import json import llm diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 73dcec6b5..76980771e 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -12,17 +12,14 @@ import llm from llm.serialization import ( - AttachmentDict, AttachmentPartDict, MessageDict, PartDict, - PromptDict, ResponseDict, ReasoningPartDict, TextPartDict, ToolCallPartDict, ToolResultPartDict, - UsageDict, ) # ---- required/optional keys ---------------------------------------- From 639c2b1309e79475e195be3c8e2ef9b6a200e720 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 22 Apr 2026 10:16:31 -0700 Subject: [PATCH 044/258] 0.31a0.dev0 release https://static.simonwillison.net/static/2026/llm-0.31a0.dev0-py3-none-any.whl https://static.simonwillison.net/static/2026/llm-0.31a0.dev0.tar.gz --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a9ae3b6dc..9e8e2e443 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm" -version = "0.30" +version = "0.31a0.dev0" description = "CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine." readme = { file = "README.md", content-type = "text/markdown" } authors = [ From eb45de6bfc1291da8120c98993391f8b744cfaec Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 22 Apr 2026 17:18:08 +0000 Subject: [PATCH 045/258] Ran cog --- docs/fragments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/fragments.md b/docs/fragments.md index f281fb063..69bb451bf 100644 --- a/docs/fragments.md +++ b/docs/fragments.md @@ -30,7 +30,7 @@ from importlib.metadata import version llm_version = version("llm") cog.out(f'The URL will be fetched with the user-agent `llm/{llm_version} (https://llm.datasette.io/)`.') ]]]--> -The URL will be fetched with the user-agent `llm/0.30 (https://llm.datasette.io/)`. +The URL will be fetched with the user-agent `llm/0.31a0.dev0 (https://llm.datasette.io/)`. The `-f` option can be used multiple times to combine together multiple fragments. From 0cc5a1cdc835730af72e7948d6bb5082b5960df5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 24 Apr 2026 12:19:16 -0700 Subject: [PATCH 046/258] Ignore non-llm folders in pyproject.toml --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a9ae3b6dc..63fc8ca68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,3 +81,6 @@ llm = "llm.cli:cli" [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["llm*"] From c9a3ac9fe0ffe5361eb86ddb50b8f941c2e4877b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 24 Apr 2026 15:41:12 -0700 Subject: [PATCH 047/258] New model: gpt-5.5 - refs #1418 --- docs/openai-models.md | 2 ++ docs/usage.md | 44 ++++++++++++++++++++++++++++ llm/default_plugins/openai_models.py | 21 +++++++++++++ 3 files changed, 67 insertions(+) diff --git a/docs/openai-models.md b/docs/openai-models.md index ce3365dc4..d77f0936a 100644 --- a/docs/openai-models.md +++ b/docs/openai-models.md @@ -75,6 +75,8 @@ OpenAI Chat: gpt-5.4-mini OpenAI Chat: gpt-5.4-mini-2026-03-17 OpenAI Chat: gpt-5.4-nano OpenAI Chat: gpt-5.4-nano-2026-03-17 +OpenAI Chat: gpt-5.5 +OpenAI Chat: gpt-5.5-2026-04-23 OpenAI Completion: gpt-3.5-turbo-instruct (aliases: 3.5-instruct, chatgpt-instruct) ``` diff --git a/docs/usage.md b/docs/usage.md index 651928907..6b13a64e3 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1427,6 +1427,50 @@ OpenAI Chat: gpt-5.4-nano-2026-03-17 Keys: key: openai env_var: OPENAI_API_KEY +OpenAI Chat: gpt-5.5 + Options: + temperature: float + max_tokens: int + top_p: float + frequency_penalty: float + presence_penalty: float + stop: str + logit_bias: dict, str + seed: int + json_object: boolean + reasoning_effort: str + Attachment types: + application/pdf, image/gif, image/jpeg, image/png, image/webp + Features: + - streaming + - schemas + - tools + - async + Keys: + key: openai + env_var: OPENAI_API_KEY +OpenAI Chat: gpt-5.5-2026-04-23 + Options: + temperature: float + max_tokens: int + top_p: float + frequency_penalty: float + presence_penalty: float + stop: str + logit_bias: dict, str + seed: int + json_object: boolean + reasoning_effort: str + Attachment types: + application/pdf, image/gif, image/jpeg, image/png, image/webp + Features: + - streaming + - schemas + - tools + - async + Keys: + key: openai + env_var: OPENAI_API_KEY OpenAI Completion: gpt-3.5-turbo-instruct (aliases: 3.5-instruct, chatgpt-instruct) Options: temperature: float diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 55fa7a38d..315c0e516 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -258,6 +258,27 @@ def register_models(register): supports_tools=True, ), ) + # GPT-5.5 + for model_id in ( + "gpt-5.5", + "gpt-5.5-2026-04-23", + ): + register( + Chat( + model_id, + vision=True, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + AsyncChat( + model_id, + vision=True, + reasoning=True, + supports_schema=True, + supports_tools=True, + ), + ) # The -instruct completion model register( From 021a29d61a574e7b807f072b1c54deb5a17c04e4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 24 Apr 2026 16:08:26 -0700 Subject: [PATCH 048/258] OpenAI verbosity option Refs https://github.com/simonw/llm/issues/1418#issuecomment-4316867527 --- docs/usage.md | 18 ++++++++ llm/default_plugins/openai_models.py | 37 ++++++++++++++- pyproject.toml | 2 +- tests/test_cli_openai_models.py | 67 ++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 2 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 6b13a64e3..2c1733273 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1085,6 +1085,7 @@ OpenAI Chat: gpt-5 stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1107,6 +1108,7 @@ OpenAI Chat: gpt-5-mini stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1129,6 +1131,7 @@ OpenAI Chat: gpt-5-nano stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1151,6 +1154,7 @@ OpenAI Chat: gpt-5-2025-08-07 stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1173,6 +1177,7 @@ OpenAI Chat: gpt-5-mini-2025-08-07 stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1195,6 +1200,7 @@ OpenAI Chat: gpt-5-nano-2025-08-07 stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1217,6 +1223,7 @@ OpenAI Chat: gpt-5.1 stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1239,6 +1246,7 @@ OpenAI Chat: gpt-5.1-chat-latest stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1261,6 +1269,7 @@ OpenAI Chat: gpt-5.2 stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1283,6 +1292,7 @@ OpenAI Chat: gpt-5.2-chat-latest stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1305,6 +1315,7 @@ OpenAI Chat: gpt-5.4 stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1327,6 +1338,7 @@ OpenAI Chat: gpt-5.4-2026-03-05 stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1349,6 +1361,7 @@ OpenAI Chat: gpt-5.4-mini stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1371,6 +1384,7 @@ OpenAI Chat: gpt-5.4-mini-2026-03-17 stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1393,6 +1407,7 @@ OpenAI Chat: gpt-5.4-nano stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1415,6 +1430,7 @@ OpenAI Chat: gpt-5.4-nano-2026-03-17 stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1437,6 +1453,7 @@ OpenAI Chat: gpt-5.5 stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: @@ -1459,6 +1476,7 @@ OpenAI Chat: gpt-5.5-2026-04-23 stop: str logit_bias: dict, str seed: int + verbosity: str json_object: boolean reasoning_effort: str Attachment types: diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 315c0e516..d4e5b2bce 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -181,6 +181,7 @@ def register_models(register): model_id, vision=True, reasoning=True, + verbosity=True, supports_schema=True, supports_tools=True, ), @@ -188,6 +189,7 @@ def register_models(register): model_id, vision=True, reasoning=True, + verbosity=True, supports_schema=True, supports_tools=True, ), @@ -202,6 +204,7 @@ def register_models(register): model_id, vision=True, reasoning=True, + verbosity=True, supports_schema=True, supports_tools=True, ), @@ -209,6 +212,7 @@ def register_models(register): model_id, vision=True, reasoning=True, + verbosity=True, supports_schema=True, supports_tools=True, ), @@ -220,6 +224,7 @@ def register_models(register): model_id, vision=True, reasoning=True, + verbosity=True, supports_schema=True, supports_tools=True, ), @@ -227,6 +232,7 @@ def register_models(register): model_id, vision=True, reasoning=True, + verbosity=True, supports_schema=True, supports_tools=True, ), @@ -247,6 +253,7 @@ def register_models(register): model_id, vision=True, reasoning=True, + verbosity=True, supports_schema=True, supports_tools=True, ), @@ -254,6 +261,7 @@ def register_models(register): model_id, vision=True, reasoning=True, + verbosity=True, supports_schema=True, supports_tools=True, ), @@ -268,6 +276,7 @@ def register_models(register): model_id, vision=True, reasoning=True, + verbosity=True, supports_schema=True, supports_tools=True, ), @@ -275,6 +284,7 @@ def register_models(register): model_id, vision=True, reasoning=True, + verbosity=True, supports_schema=True, supports_tools=True, ), @@ -547,6 +557,12 @@ class ReasoningEffortEnum(str, Enum): xhigh = "xhigh" +class VerbosityEnum(str, Enum): + low = "low" + medium = "medium" + high = "high" + + class OptionsForReasoning(SharedOptions): json_object: Optional[bool] = Field( description="Output a valid JSON object {...}. Prompt must mention JSON.", @@ -562,6 +578,20 @@ class OptionsForReasoning(SharedOptions): ) +class OptionsWithVerbosity(SharedOptions): + verbosity: Optional[VerbosityEnum] = Field( + description=( + "Controls how verbose the model's response should be. Supported values " + "are low, medium, and high." + ), + default=None, + ) + + +class OptionsForReasoningAndVerbosity(OptionsForReasoning, OptionsWithVerbosity): + pass + + def _attachment(attachment): url = attachment.url base64_content = "" @@ -606,6 +636,7 @@ def __init__( vision=False, audio=False, reasoning=False, + verbosity=False, supports_schema=False, supports_tools=False, allows_system_prompt=True, @@ -626,8 +657,12 @@ def __init__( self.attachment_types = set() - if reasoning: + if reasoning and verbosity: + self.Options = OptionsForReasoningAndVerbosity + elif reasoning: self.Options = OptionsForReasoning + elif verbosity: + self.Options = OptionsWithVerbosity if vision: self.attachment_types.update( diff --git a/pyproject.toml b/pyproject.toml index 63fc8ca68..b6a80b3b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ dependencies = [ "click", "condense-json>=0.1.3", - "openai>=1.55.3", + "openai>=2.32.0", "click-default-group>=1.2.3", "sqlite-utils>=3.37", "sqlite-migrate>=0.1a2", diff --git a/tests/test_cli_openai_models.py b/tests/test_cli_openai_models.py index fbb382de7..1a7da760b 100644 --- a/tests/test_cli_openai_models.py +++ b/tests/test_cli_openai_models.py @@ -1,4 +1,6 @@ from click.testing import CliRunner +import json +import llm from llm.cli import cli import pytest import sqlite_utils @@ -59,6 +61,71 @@ def test_openai_options_min_max(): assert f"less than or equal to {max_val}" in result2.output +@pytest.mark.parametrize( + "model_id", + ( + "gpt-5", + "gpt-5-mini", + "gpt-5.1", + "gpt-5.2", + "gpt-5.4", + "gpt-5.5", + ), +) +def test_gpt5_models_support_verbosity_option(model_id): + assert "verbosity" in llm.get_model(model_id).Options.model_fields + assert "verbosity" in llm.get_async_model(model_id).Options.model_fields + + +@pytest.mark.parametrize("model_id", ("gpt-4o", "gpt-4.5-preview", "o3", "o4-mini")) +def test_non_gpt5_openai_chat_models_do_not_support_verbosity_option(model_id): + assert "verbosity" not in llm.get_model(model_id).Options.model_fields + assert "verbosity" not in llm.get_async_model(model_id).Options.model_fields + + +def test_gpt5_verbosity_option_is_sent_to_openai_chat_completions(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "model": "gpt-5", + "usage": {}, + "choices": [{"message": {"content": "Verbose enough"}}], + }, + headers={"Content-Type": "application/json"}, + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "-m", + "gpt-5", + "-o", + "verbosity", + "high", + "--no-stream", + "--key", + "x", + "Say hi", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["verbosity"] == "high" + assert "text" not in request_body + + +def test_gpt5_verbosity_option_validates_allowed_values(): + runner = CliRunner() + result = runner.invoke( + cli, + ["-m", "gpt-5", "-o", "verbosity", "extreme", "Say hi"], + ) + assert result.exit_code == 1 + assert "Input should be 'low', 'medium' or 'high'" in result.output + + @pytest.mark.parametrize("model", ("gpt-4o-mini", "gpt-4o-audio-preview")) @pytest.mark.parametrize("filetype", ("mp3", "wav")) def test_only_gpt4_audio_preview_allows_mp3_or_wav(httpx_mock, model, filetype): From 87efea179d55484c6a2fcc205c36afd7a882808a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 24 Apr 2026 16:11:22 -0700 Subject: [PATCH 049/258] Cleaner dynamic self.Options building, refs #1418 --- llm/default_plugins/openai_models.py | 74 +++++++++++++++------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index d4e5b2bce..86cafef2f 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -23,7 +23,7 @@ import openai import os -from pydantic import field_validator, Field +from pydantic import create_model, field_validator, Field from typing import AsyncGenerator, cast, List, Iterable, Iterator, Optional, Union import json @@ -563,33 +563,41 @@ class VerbosityEnum(str, Enum): high = "high" -class OptionsForReasoning(SharedOptions): - json_object: Optional[bool] = Field( - description="Output a valid JSON object {...}. Prompt must mention JSON.", - default=None, - ) - reasoning_effort: Optional[ReasoningEffortEnum] = Field( - description=( - "Constraints effort on reasoning for reasoning models. Currently supported " - "values are low, medium, and high. Reducing reasoning effort can result in " - "faster responses and fewer tokens used on reasoning in a response." - ), - default=None, - ) - - -class OptionsWithVerbosity(SharedOptions): - verbosity: Optional[VerbosityEnum] = Field( - description=( - "Controls how verbose the model's response should be. Supported values " - "are low, medium, and high." - ), - default=None, - ) - - -class OptionsForReasoningAndVerbosity(OptionsForReasoning, OptionsWithVerbosity): - pass +def build_options_class(*, reasoning=False, verbosity=False): + fields = { + "json_object": ( + Optional[bool], + Field( + description="Output a valid JSON object {...}. Prompt must mention JSON.", + default=None, + ), + ) + } + if reasoning: + fields["reasoning_effort"] = ( + Optional[ReasoningEffortEnum], + Field( + description=( + "Constraints effort on reasoning for reasoning models. Currently " + "supported values are low, medium, and high. Reducing reasoning " + "effort can result in faster responses and fewer tokens used on " + "reasoning in a response." + ), + default=None, + ), + ) + if verbosity: + fields["verbosity"] = ( + Optional[VerbosityEnum], + Field( + description=( + "Controls how verbose the model's response should be. Supported " + "values are low, medium, and high." + ), + default=None, + ), + ) + return create_model("Options", __base__=SharedOptions, **fields) def _attachment(attachment): @@ -657,12 +665,10 @@ def __init__( self.attachment_types = set() - if reasoning and verbosity: - self.Options = OptionsForReasoningAndVerbosity - elif reasoning: - self.Options = OptionsForReasoning - elif verbosity: - self.Options = OptionsWithVerbosity + if reasoning or verbosity: + self.Options = build_options_class( + reasoning=reasoning, verbosity=verbosity + ) if vision: self.attachment_types.update( From 706852ecead35f2de23b5a9086814282f0c7bad2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 24 Apr 2026 16:22:43 -0700 Subject: [PATCH 050/258] New image_detail low/high/auto/original option Refs https://github.com/simonw/llm/issues/1418#issuecomment-4316983472 --- docs/usage.md | 84 +++++++++++++++----- llm/default_plugins/openai_models.py | 93 +++++++++++++++++----- tests/test_cli_openai_models.py | 111 +++++++++++++++++++++++++++ 3 files changed, 250 insertions(+), 38 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 2c1733273..e6f78a968 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -549,6 +549,9 @@ OpenAI Chat: gpt-4o (aliases: 4o) Integer seed to attempt to sample deterministically json_object: boolean Output a valid JSON object {...}. Prompt must mention JSON. + image_detail: str + Controls the detail level for image attachments. Supported values are + low, high, and auto. Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -570,6 +573,7 @@ OpenAI Chat: chatgpt-4o-latest (aliases: chatgpt-4o) logit_bias: dict, str seed: int json_object: boolean + image_detail: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -589,6 +593,7 @@ OpenAI Chat: gpt-4o-mini (aliases: 4o-mini) logit_bias: dict, str seed: int json_object: boolean + image_detail: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -610,6 +615,7 @@ OpenAI Chat: gpt-4o-audio-preview logit_bias: dict, str seed: int json_object: boolean + image_detail: str Attachment types: audio/mpeg, audio/wav Features: @@ -629,6 +635,7 @@ OpenAI Chat: gpt-4o-audio-preview-2024-12-17 logit_bias: dict, str seed: int json_object: boolean + image_detail: str Attachment types: audio/mpeg, audio/wav Features: @@ -648,6 +655,7 @@ OpenAI Chat: gpt-4o-audio-preview-2024-10-01 logit_bias: dict, str seed: int json_object: boolean + image_detail: str Attachment types: audio/mpeg, audio/wav Features: @@ -667,6 +675,7 @@ OpenAI Chat: gpt-4o-mini-audio-preview logit_bias: dict, str seed: int json_object: boolean + image_detail: str Attachment types: audio/mpeg, audio/wav Features: @@ -686,6 +695,7 @@ OpenAI Chat: gpt-4o-mini-audio-preview-2024-12-17 logit_bias: dict, str seed: int json_object: boolean + image_detail: str Attachment types: audio/mpeg, audio/wav Features: @@ -705,6 +715,7 @@ OpenAI Chat: gpt-4.1 (aliases: 4.1) logit_bias: dict, str seed: int json_object: boolean + image_detail: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -726,6 +737,7 @@ OpenAI Chat: gpt-4.1-mini (aliases: 4.1-mini) logit_bias: dict, str seed: int json_object: boolean + image_detail: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -747,6 +759,7 @@ OpenAI Chat: gpt-4.1-nano (aliases: 4.1-nano) logit_bias: dict, str seed: int json_object: boolean + image_detail: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -768,6 +781,7 @@ OpenAI Chat: gpt-3.5-turbo (aliases: 3.5, chatgpt) logit_bias: dict, str seed: int json_object: boolean + image_detail: str Features: - streaming - async @@ -785,6 +799,7 @@ OpenAI Chat: gpt-3.5-turbo-16k (aliases: chatgpt-16k, 3.5-16k) logit_bias: dict, str seed: int json_object: boolean + image_detail: str Features: - streaming - async @@ -802,6 +817,7 @@ OpenAI Chat: gpt-4 (aliases: 4, gpt4) logit_bias: dict, str seed: int json_object: boolean + image_detail: str Features: - streaming - async @@ -819,6 +835,7 @@ OpenAI Chat: gpt-4-32k (aliases: 4-32k) logit_bias: dict, str seed: int json_object: boolean + image_detail: str Features: - streaming - async @@ -836,6 +853,7 @@ OpenAI Chat: gpt-4-1106-preview logit_bias: dict, str seed: int json_object: boolean + image_detail: str Features: - streaming - async @@ -853,6 +871,7 @@ OpenAI Chat: gpt-4-0125-preview logit_bias: dict, str seed: int json_object: boolean + image_detail: str Features: - streaming - async @@ -870,6 +889,7 @@ OpenAI Chat: gpt-4-turbo-2024-04-09 logit_bias: dict, str seed: int json_object: boolean + image_detail: str Features: - streaming - async @@ -887,6 +907,7 @@ OpenAI Chat: gpt-4-turbo (aliases: gpt-4-turbo-preview, 4-turbo, 4t) logit_bias: dict, str seed: int json_object: boolean + image_detail: str Features: - streaming - async @@ -904,6 +925,7 @@ OpenAI Chat: gpt-4.5-preview-2025-02-27 logit_bias: dict, str seed: int json_object: boolean + image_detail: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -925,6 +947,7 @@ OpenAI Chat: gpt-4.5-preview (aliases: gpt-4.5) logit_bias: dict, str seed: int json_object: boolean + image_detail: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -946,6 +969,7 @@ OpenAI Chat: o1 logit_bias: dict, str seed: int json_object: boolean + image_detail: str reasoning_effort: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp @@ -967,6 +991,7 @@ OpenAI Chat: o1-2024-12-17 logit_bias: dict, str seed: int json_object: boolean + image_detail: str reasoning_effort: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp @@ -988,6 +1013,7 @@ OpenAI Chat: o1-preview logit_bias: dict, str seed: int json_object: boolean + image_detail: str Features: - streaming - async @@ -1005,6 +1031,7 @@ OpenAI Chat: o1-mini logit_bias: dict, str seed: int json_object: boolean + image_detail: str Features: - streaming - async @@ -1022,6 +1049,7 @@ OpenAI Chat: o3-mini logit_bias: dict, str seed: int json_object: boolean + image_detail: str reasoning_effort: str Features: - streaming @@ -1042,6 +1070,7 @@ OpenAI Chat: o3 logit_bias: dict, str seed: int json_object: boolean + image_detail: str reasoning_effort: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp @@ -1064,6 +1093,7 @@ OpenAI Chat: o4-mini logit_bias: dict, str seed: int json_object: boolean + image_detail: str reasoning_effort: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp @@ -1085,9 +1115,10 @@ OpenAI Chat: gpt-5 stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1108,9 +1139,10 @@ OpenAI Chat: gpt-5-mini stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1131,9 +1163,10 @@ OpenAI Chat: gpt-5-nano stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1154,9 +1187,10 @@ OpenAI Chat: gpt-5-2025-08-07 stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1177,9 +1211,10 @@ OpenAI Chat: gpt-5-mini-2025-08-07 stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1200,9 +1235,10 @@ OpenAI Chat: gpt-5-nano-2025-08-07 stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1223,9 +1259,10 @@ OpenAI Chat: gpt-5.1 stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1246,9 +1283,10 @@ OpenAI Chat: gpt-5.1-chat-latest stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1269,9 +1307,10 @@ OpenAI Chat: gpt-5.2 stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1292,9 +1331,10 @@ OpenAI Chat: gpt-5.2-chat-latest stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1315,9 +1355,10 @@ OpenAI Chat: gpt-5.4 stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1338,9 +1379,10 @@ OpenAI Chat: gpt-5.4-2026-03-05 stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1361,9 +1403,10 @@ OpenAI Chat: gpt-5.4-mini stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1384,9 +1427,10 @@ OpenAI Chat: gpt-5.4-mini-2026-03-17 stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1407,9 +1451,10 @@ OpenAI Chat: gpt-5.4-nano stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1430,9 +1475,10 @@ OpenAI Chat: gpt-5.4-nano-2026-03-17 stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1453,9 +1499,10 @@ OpenAI Chat: gpt-5.5 stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1476,9 +1523,10 @@ OpenAI Chat: gpt-5.5-2026-04-23 stop: str logit_bias: dict, str seed: int - verbosity: str json_object: boolean + image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 86cafef2f..7c590d8b3 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -254,6 +254,7 @@ def register_models(register): vision=True, reasoning=True, verbosity=True, + image_detail_original=True, supports_schema=True, supports_tools=True, ), @@ -262,6 +263,7 @@ def register_models(register): vision=True, reasoning=True, verbosity=True, + image_detail_original=True, supports_schema=True, supports_tools=True, ), @@ -277,6 +279,7 @@ def register_models(register): vision=True, reasoning=True, verbosity=True, + image_detail_original=True, supports_schema=True, supports_tools=True, ), @@ -285,6 +288,7 @@ def register_models(register): vision=True, reasoning=True, verbosity=True, + image_detail_original=True, supports_schema=True, supports_tools=True, ), @@ -563,7 +567,29 @@ class VerbosityEnum(str, Enum): high = "high" -def build_options_class(*, reasoning=False, verbosity=False): +class ImageDetailEnum(str, Enum): + low = "low" + high = "high" + auto = "auto" + + +class ImageDetailWithOriginalEnum(str, Enum): + low = "low" + high = "high" + original = "original" + auto = "auto" + + +def enum_values_sentence(enum_class): + values = [item.value for item in enum_class] + if len(values) == 1: + return values[0] + return "{}, and {}".format(", ".join(values[:-1]), values[-1]) + + +def build_options_class( + *, reasoning=False, verbosity=False, image_detail_original=False +): fields = { "json_object": ( Optional[bool], @@ -573,6 +599,20 @@ def build_options_class(*, reasoning=False, verbosity=False): ), ) } + image_detail_enum = ( + ImageDetailWithOriginalEnum if image_detail_original else ImageDetailEnum + ) + image_detail_values = enum_values_sentence(image_detail_enum) + fields["image_detail"] = ( + Optional[image_detail_enum], + Field( + description=( + "Controls the detail level for image attachments. Supported values are " + f"{image_detail_values}." + ), + default=None, + ), + ) if reasoning: fields["reasoning_effort"] = ( Optional[ReasoningEffortEnum], @@ -600,7 +640,7 @@ def build_options_class(*, reasoning=False, verbosity=False): return create_model("Options", __base__=SharedOptions, **fields) -def _attachment(attachment): +def _attachment(attachment, image_detail=None): url = attachment.url base64_content = "" if not url or attachment.resolve_type().startswith("audio/"): @@ -617,7 +657,10 @@ def _attachment(attachment): }, } if attachment.resolve_type().startswith("image/"): - return {"type": "image_url", "image_url": {"url": url}} + image_url = {"url": url} + if image_detail: + image_url["detail"] = image_detail + return {"type": "image_url", "image_url": image_url} else: format_ = "wav" if attachment.resolve_type() == "audio/wav" else "mp3" return { @@ -645,6 +688,7 @@ def __init__( audio=False, reasoning=False, verbosity=False, + image_detail_original=False, supports_schema=False, supports_tools=False, allows_system_prompt=True, @@ -665,9 +709,11 @@ def __init__( self.attachment_types = set() - if reasoning or verbosity: + if reasoning or verbosity or image_detail_original: self.Options = build_options_class( - reasoning=reasoning, verbosity=verbosity + reasoning=reasoning, + verbosity=verbosity, + image_detail_original=image_detail_original, ) if vision: @@ -692,8 +738,10 @@ def __init__( def __str__(self) -> str: return "OpenAI Chat: {}".format(self.model_id) - def build_messages(self, prompt, conversation): + def build_messages(self, prompt, conversation, image_detail=None): messages = [] + if image_detail is not None: + image_detail = image_detail.value current_system = None if conversation is not None: for prev_response in conversation.responses: @@ -712,7 +760,9 @@ def build_messages(self, prompt, conversation): {"type": "text", "text": prev_response.prompt.prompt} ) for attachment in prev_response.attachments: - attachment_message.append(_attachment(attachment)) + attachment_message.append( + _attachment(attachment, image_detail=image_detail) + ) messages.append({"role": "user", "content": attachment_message}) elif prev_response.prompt.prompt: messages.append( @@ -765,7 +815,9 @@ def build_messages(self, prompt, conversation): if prompt.prompt: attachment_message.append({"type": "text", "text": prompt.prompt}) for attachment in prompt.attachments: - attachment_message.append(_attachment(attachment)) + attachment_message.append( + _attachment(attachment, image_detail=image_detail) + ) messages.append({"role": "user", "content": attachment_message}) return messages @@ -807,6 +859,7 @@ def get_client(self, key, *, async_=False): def build_kwargs(self, prompt, stream): kwargs = dict(not_nulls(prompt.options)) json_object = kwargs.pop("json_object", None) + kwargs.pop("image_detail", None) if "max_tokens" not in kwargs and self.default_max_tokens is not None: kwargs["max_tokens"] = self.default_max_tokens if json_object: @@ -838,11 +891,7 @@ class Chat(_Shared, KeyModel): key_env_var = "OPENAI_API_KEY" default_max_tokens = None - class Options(SharedOptions): - json_object: Optional[bool] = Field( - description="Output a valid JSON object {...}. Prompt must mention JSON.", - default=None, - ) + Options = build_options_class() def execute( self, @@ -854,7 +903,11 @@ def execute( ) -> Iterator[str]: if prompt.system and not self.allows_system_prompt: raise NotImplementedError("Model does not support system prompts") - messages = self.build_messages(prompt, conversation) + messages = self.build_messages( + prompt, + conversation, + image_detail=getattr(prompt.options, "image_detail", None), + ) kwargs = self.build_kwargs(prompt, stream) client = self.get_client(key) usage = None @@ -928,11 +981,7 @@ class AsyncChat(_Shared, AsyncKeyModel): key_env_var = "OPENAI_API_KEY" default_max_tokens = None - class Options(SharedOptions): - json_object: Optional[bool] = Field( - description="Output a valid JSON object {...}. Prompt must mention JSON.", - default=None, - ) + Options = build_options_class() async def execute( self, @@ -944,7 +993,11 @@ async def execute( ) -> AsyncGenerator[str, None]: if prompt.system and not self.allows_system_prompt: raise NotImplementedError("Model does not support system prompts") - messages = self.build_messages(prompt, conversation) + messages = self.build_messages( + prompt, + conversation, + image_detail=getattr(prompt.options, "image_detail", None), + ) kwargs = self.build_kwargs(prompt, stream) client = self.get_client(key, async_=True) usage = None diff --git a/tests/test_cli_openai_models.py b/tests/test_cli_openai_models.py index 1a7da760b..eabc7d84d 100644 --- a/tests/test_cli_openai_models.py +++ b/tests/test_cli_openai_models.py @@ -126,6 +126,117 @@ def test_gpt5_verbosity_option_validates_allowed_values(): assert "Input should be 'low', 'medium' or 'high'" in result.output +@pytest.mark.parametrize( + "model_id,expected_description", + ( + ( + "gpt-4o", + "Controls the detail level for image attachments. Supported values are low, high, and auto.", + ), + ( + "gpt-5.4", + "Controls the detail level for image attachments. Supported values are low, high, original, and auto.", + ), + ( + "gpt-5.5", + "Controls the detail level for image attachments. Supported values are low, high, original, and auto.", + ), + ), +) +def test_openai_image_detail_option_description(model_id, expected_description): + field = llm.get_model(model_id).Options.model_fields["image_detail"] + assert field.description == expected_description + + +def test_openai_image_detail_option_is_sent_on_image_attachments(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "model": "gpt-4o", + "usage": {}, + "choices": [{"message": {"content": "Looks detailed"}}], + }, + headers={"Content-Type": "application/json"}, + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "-m", + "gpt-4o", + "-o", + "image_detail", + "high", + "--at", + "https://example.com/image.jpg", + "image/jpeg", + "--no-stream", + "--key", + "x", + "Describe this", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[-1].content) + image_part = request_body["messages"][0]["content"][1] + assert image_part == { + "type": "image_url", + "image_url": { + "url": "https://example.com/image.jpg", + "detail": "high", + }, + } + assert "image_detail" not in request_body + + +def test_openai_image_detail_original_is_sent_for_gpt54(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "model": "gpt-5.4", + "usage": {}, + "choices": [{"message": {"content": "Original detail"}}], + }, + headers={"Content-Type": "application/json"}, + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "-m", + "gpt-5.4", + "-o", + "image_detail", + "original", + "--at", + "https://example.com/image.jpg", + "image/jpeg", + "--no-stream", + "--key", + "x", + "Describe this", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[-1].content) + image_part = request_body["messages"][0]["content"][1] + assert image_part["image_url"]["detail"] == "original" + + +def test_openai_image_detail_original_is_rejected_for_other_models(): + runner = CliRunner() + result = runner.invoke( + cli, + ["-m", "gpt-5", "-o", "image_detail", "original", "Say hi"], + ) + assert result.exit_code == 1 + assert "Input should be 'low', 'high' or 'auto'" in result.output + + @pytest.mark.parametrize("model", ("gpt-4o-mini", "gpt-4o-audio-preview")) @pytest.mark.parametrize("filetype", ("mp3", "wav")) def test_only_gpt4_audio_preview_allows_mp3_or_wav(httpx_mock, model, filetype): From 5ce40fd70359a5966466b20998aaabd21a6212c6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 24 Apr 2026 16:31:18 -0700 Subject: [PATCH 051/258] Release 0.31 Refs #1418 --- docs/changelog.md | 8 ++++++++ docs/fragments.md | 2 +- pyproject.toml | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 3f4241773..563ec4133 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,13 @@ # Changelog +(v0_31)= +## 0.31 (2026-04-24) + +- New GPT-5.5 OpenAI model: `llm -m gpt-5.5`. [#1418](https://github.com/simonw/llm/issues/1418) +- New option to set the [text verbosity level](https://developers.openai.com/cookbook/examples/gpt-5/gpt-5_new_params_and_tools#1-verbosity-parameter) for GPT-5+ OpenAI models: `-o verbosity low`. Values are `low`, `medium`, `high`. +- New option for setting the [image detail level](https://developers.openai.com/api/docs/guides/images-vision#choose-an-image-detail-level) used for image attachments to OpenAI models: `-o image_detail low` - values are `low`, `high` and `auto`, and GPT-5.4 and 5.5 also accept `original`. +- Models listed in `extra-openai-models.yaml` are now also registered as asynchronous. [#1395](https://github.com/simonw/llm/issues/1395) + (v0_30)= ## 0.30 (2026-03-31) diff --git a/docs/fragments.md b/docs/fragments.md index f281fb063..a7e908a9a 100644 --- a/docs/fragments.md +++ b/docs/fragments.md @@ -30,7 +30,7 @@ from importlib.metadata import version llm_version = version("llm") cog.out(f'The URL will be fetched with the user-agent `llm/{llm_version} (https://llm.datasette.io/)`.') ]]]--> -The URL will be fetched with the user-agent `llm/0.30 (https://llm.datasette.io/)`. +The URL will be fetched with the user-agent `llm/0.31 (https://llm.datasette.io/)`. The `-f` option can be used multiple times to combine together multiple fragments. diff --git a/pyproject.toml b/pyproject.toml index b6a80b3b5..dc73954d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm" -version = "0.30" +version = "0.31" description = "CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine." readme = { file = "README.md", content-type = "text/markdown" } authors = [ From 3b0d0fa0a5151e575a0ba0f541e12eace207862e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 09:35:48 -0700 Subject: [PATCH 052/258] part_index is now (mostly) automatically assigned --- docs/plugins/advanced-model-plugins.md | 102 ++++------- llm/default_plugins/openai_models.py | 29 +-- llm/models.py | 238 ++++++++++++++++--------- llm/parts.py | 12 +- tests/test_parts.py | 198 ++++++++++++++++++++ 5 files changed, 394 insertions(+), 185 deletions(-) diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index 8e65641ca..cdae9d087 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -258,7 +258,7 @@ The 0.31 alpha introduced a richer contract for plugins than "yield strings": 2. **`build_messages` (or equivalent) reads `prompt.messages`** — a `list[llm.Message]` that is the complete input chain for this turn. 3. **Opaque provider tokens round-trip via `provider_metadata`** — Anthropic thinking signatures, Gemini thought signatures, OpenAI Responses API encrypted reasoning blobs. Plugins stash whatever the API returns, then echo it back on the next request. -**Backward compatibility is guaranteed.** A plugin that still yields plain `str` from `execute()` works unchanged — each string is wrapped as a `StreamEvent(type="text", chunk=..., part_index=0)` internally. +**Backward compatibility is guaranteed.** A plugin that still yields plain `str` from `execute()` works unchanged — each string is wrapped as a `StreamEvent(type="text", chunk=...)` internally. ### Yielding StreamEvent from execute() @@ -271,71 +271,53 @@ def execute(self, prompt, stream, response, conversation, key=None): for chunk in provider_sdk.stream(...): if chunk.type == "text": - yield StreamEvent(type="text", chunk=chunk.text, part_index=0) + yield StreamEvent(type="text", chunk=chunk.text) elif chunk.type == "thinking": - yield StreamEvent(type="reasoning", chunk=chunk.text, part_index=0) + yield StreamEvent(type="reasoning", chunk=chunk.text) ``` -A `StreamEvent` has five frequently-used fields: +That's the whole pattern for most plugins. The framework figures out which events group into which Part. + +A `StreamEvent` has four frequently-used fields: - **`type`** — one of `"text"`, `"reasoning"`, `"tool_call_name"`, `"tool_call_args"`, `"tool_result"`. - **`chunk`** — the text fragment. For tool calls this is the tool name (for `tool_call_name`) or a partial JSON string (for `tool_call_args`). -- **`part_index`** — a monotonically allocated integer identifying which `Part` this event contributes to. All events sharing a `part_index` must belong to the same family; events at the same index concatenate into one Part. -- **`tool_call_id`** — the provider's id for the tool call, set on `tool_call_name` / `tool_call_args` / `tool_result` events. +- **`tool_call_id`** — the provider's id for the tool call, set on `tool_call_name` / `tool_call_args` / `tool_result` events. Also the signal the framework uses to group tool-call events into one `ToolCallPart`. - **`provider_metadata`** — an optional `dict[str, dict]` namespaced by provider name. Carries opaque data (signatures, encrypted blobs) that must be echoed back on future requests. -Two additional fields exist for special cases: +Three additional fields exist for special cases: - **`server_executed: bool`** — set `True` for server-side tool calls (for example, Anthropic web search) and their results. The model ran the tool internally. - **`tool_name`** — set on `tool_result` events to identify which tool this result came from. +- **`part_index: int | None`** — defaults to `None`, which means "let the framework decide which Part this event belongs to." Pass an explicit integer only when you need to override the default grouping (see [below](#part-index-overrides)). -### Allocating `part_index` +### How events group into Parts -`part_index` groups events into Parts. Rules: +When you leave `part_index` as `None` (the default), the framework groups events using these rules: -- **Same `part_index` + same family** → events are appended into one Part (text concatenates; tool-call args accumulate into the final JSON). -- **Same `part_index` + different family** → the framework raises `ValueError`. That's a plugin bug — allocate a new index when a new content block begins. -- **Tool calls span two event types**: `tool_call_name` and `tool_call_args` at the *same* `part_index` combine into one `ToolCallPart`. The name arrives first; the args stream in as partial JSON and are parsed when the part finalizes. +- **Consecutive same-family events concatenate.** Two `text` events in a row become one `TextPart`. Two `reasoning` events in a row become one `ReasoningPart`. A family transition (text → reasoning, or reasoning → text) starts a new Part. +- **Tool calls group by `tool_call_id`.** A `tool_call_name` and any number of `tool_call_args` events sharing a `tool_call_id` combine into one `ToolCallPart` — even if they're interleaved with other events (parallel tool calls). +- **`tool_result` is always its own Part**, paired to the originating call by `tool_call_id`. -A typical allocation scheme: +This handles every common shape without a plugin-side allocator: -``` -part_index=0 reasoning chunks -part_index=1 text chunks -part_index=2 first tool_call (name + streaming args) -part_index=3 second tool_call (parallel) -``` +| Stream | Resulting Parts | +|-------------------------------------------|----------------------------------------------------------| +| `text` × N | one `TextPart` | +| `reasoning` × N, then `text` × N | `ReasoningPart`, `TextPart` | +| `text`, `tool_call_name`+`args`, `text` | `TextPart`, `ToolCallPart`, `TextPart` | +| Parallel tool calls (interleaved by id) | one `ToolCallPart` per distinct `tool_call_id` | +| `reasoning`, tool call, `reasoning` | `ReasoningPart`, `ToolCallPart`, `ReasoningPart` | -For providers that emit discrete content blocks (like Anthropic's `content_block_start` / `content_block_delta` events), a natural implementation is a dict keyed by block index: +(part-index-overrides)= +### Setting `part_index` explicitly -```python -state = {"blocks": {}, "next_part_index": 0} - -# On content_block_start: -idx = event.index -pi = state["next_part_index"] -state["next_part_index"] += 1 -state["blocks"][idx] = {"kind": block.type, "part_index": pi} - -# On content_block_delta: -info = state["blocks"][event.index] -pi = info["part_index"] -yield StreamEvent(type="text", chunk=delta.text, part_index=pi) -``` +In rare cases you'll want to override the default grouping: -For providers that emit discrete parts per streamed chunk without start/stop markers (like Gemini), track the current block kind and advance the index on kind changes: +- **Forcing a single TextPart across non-adjacent text bursts.** If your provider interleaves text deltas with tool calls but you want all the text concatenated into one `TextPart`, pass `part_index=0` on every text event. (The default behavior produces separate `TextPart`s on each side of the tool calls — usually what you want, but not always.) +- **Tool-call args arriving before the id.** If your provider streams args before the `tool_call_id` is known, assign your own index per logical tool call and pass it on each event of that call. -```python -state = {"index": 0, "kind": None} - -def allocate_for_kind(state, new_kind): - if state["kind"] == new_kind: - return state["index"] # concat - if state["kind"] is not None: - state["index"] += 1 # advance past previous block - state["kind"] = new_kind - return state["index"] -``` +You can mix explicit indices with `None` in the same stream — the framework reserves your explicit values and decides the rest. ### Reasoning tokens @@ -344,10 +326,10 @@ Two modes are supported: **Streamed reasoning text** (Anthropic extended thinking, Gemini with `includeThoughts: true`): ```python -yield StreamEvent(type="reasoning", chunk=thinking_chunk, part_index=0) +yield StreamEvent(type="reasoning", chunk=thinking_chunk) ``` -Text events and reasoning events at different indexes produce distinct `TextPart` and `ReasoningPart` entries in `response.messages`. +Reasoning events that appear before/after text events become distinct `ReasoningPart` and `TextPart` entries in `response.messages` automatically. If your provider emits two thinking blocks separated by a tool call, you'll get two `ReasoningPart`s — exactly what you want. **Opaque reasoning token count** (OpenAI o-series, Gemini without `includeThoughts`): @@ -363,25 +345,23 @@ For OpenAI this count lives in `usage.completion_tokens_details.reasoning_tokens ### Tool calls -Each tool call emits two event types at the same `part_index`: +Each tool call emits two event types sharing a `tool_call_id`: ```python yield StreamEvent( type="tool_call_name", chunk=tool_name, - part_index=tc_part_index, tool_call_id=tool_call_id, ) # then, as the provider streams JSON args: yield StreamEvent( type="tool_call_args", chunk=partial_json_fragment, - part_index=tc_part_index, tool_call_id=tool_call_id, ) ``` -Some providers (Gemini) emit the complete tool call in one chunk — fine; emit both events back-to-back with the full name and full JSON. +The framework groups them by `tool_call_id` — so parallel tool calls (where args for tool A and tool B interleave on the wire) just work without any per-call index tracking. Some providers (Gemini) emit the complete tool call in one chunk — fine; emit both events back-to-back with the full name and full JSON. For client-side tool calls — tools that LLM should execute locally in a chain — **also call `response.add_tool_call()`**. The chain-execution path (`response.tool_calls()` → `execute_tool_calls()`) reads from the explicitly-added list, not from the StreamEvent buffer. Your code should do both: @@ -403,14 +383,12 @@ For tools the API executes internally, set `server_executed=True` on the events. yield StreamEvent( type="tool_call_name", chunk="web_search", - part_index=tc_pi, tool_call_id=tool_id, server_executed=True, ) yield StreamEvent( type="tool_call_args", chunk=json.dumps(query_args), - part_index=tc_pi, tool_call_id=tool_id, server_executed=True, ) @@ -422,7 +400,6 @@ The tool *result* (for example, the search hits) is also emitted as an event: yield StreamEvent( type="tool_result", chunk=human_readable_summary, - part_index=tr_pi, tool_call_id=tool_id, server_executed=True, tool_name="web_search", @@ -442,7 +419,7 @@ Some providers require you to echo back opaque fields on the next request for mu - **Google Gemini** — `thoughtSignature` on `functionCall` parts when thinking is active. - **OpenAI Responses API** — `encrypted_content` on reasoning items in stateless mode. -These values are attached to a `StreamEvent` via its `provider_metadata` field. The framework merges metadata across events at the same `part_index` (last non-None wins per top-level key) and persists it on the finalized Part. +These values are attached to a `StreamEvent` via its `provider_metadata` field. The framework merges metadata across events that group into the same Part (last non-None wins per top-level key) and persists it on the finalized Part. Namespace under your provider's name so transcripts that mix providers don't collide: @@ -451,7 +428,6 @@ Namespace under your provider's name so transcripts that mix providers don't col yield StreamEvent( type="reasoning", chunk="", - part_index=reasoning_pi, provider_metadata={"anthropic": {"signature": sig}}, ) ``` @@ -461,7 +437,6 @@ yield StreamEvent( yield StreamEvent( type="tool_call_name", chunk=name, - part_index=tc_pi, tool_call_id=tc_id, provider_metadata={"gemini": {"thoughtSignature": sig}}, ) @@ -471,39 +446,32 @@ Treat other providers' entries as opaque; don't parse them. The framework round- ### Non-streaming path -When `stream=False` (or the provider returns a complete message at once), emit one event per content block: +When `stream=False` (or the provider returns a complete message at once), emit one event per content block — no index tracking required: ```python else: completion = client.messages.create(**kwargs) response.response_json = completion.model_dump() - pi = 0 for block in completion.content: if block.type == "thinking": yield StreamEvent( type="reasoning", chunk=block.thinking, - part_index=pi, provider_metadata={"anthropic": {"signature": block.signature}}, ) - pi += 1 elif block.type == "text": - yield StreamEvent(type="text", chunk=block.text, part_index=pi) - pi += 1 + yield StreamEvent(type="text", chunk=block.text) elif block.type == "tool_use": yield StreamEvent( type="tool_call_name", chunk=block.name, - part_index=pi, tool_call_id=block.id, ) yield StreamEvent( type="tool_call_args", chunk=json.dumps(block.input), - part_index=pi, tool_call_id=block.id, ) - pi += 1 ``` ## Consuming prompt.messages in build_messages diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index cf28e810d..fe346b65e 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -948,11 +948,6 @@ def execute( ) chunks = [] tool_calls = {} - # part_index allocator. Text events always use 0, so tool - # calls start at 1 and keep a stable index per OpenAI delta - # index. This keeps _build_parts from mixing families. - tc_part_index = {} - next_part_index = 1 for chunk in completion: chunks.append(chunk) if chunk.usage: @@ -964,12 +959,9 @@ def execute( idx = tool_call.index if idx not in tool_calls: tool_calls[idx] = tool_call - tc_part_index[idx] = next_part_index - next_part_index += 1 yield StreamEvent( type="tool_call_name", chunk=tool_call.function.name or "", - part_index=tc_part_index[idx], tool_call_id=tool_call.id, ) else: @@ -980,7 +972,6 @@ def execute( yield StreamEvent( type="tool_call_args", chunk=tool_call.function.arguments, - part_index=tc_part_index[idx], tool_call_id=tool_calls[idx].id, ) try: @@ -990,7 +981,7 @@ def execute( if content: # Empty strings are noise (OpenAI's first chunk # with role=assistant has content=""). - yield StreamEvent(type="text", chunk=content, part_index=0) + yield StreamEvent(type="text", chunk=content) response.response_json = remove_dict_none_values(combine_chunks(chunks)) if tool_calls: for value in tool_calls.values(): @@ -1010,7 +1001,6 @@ def execute( ) usage = completion.usage.model_dump() response.response_json = remove_dict_none_values(completion.model_dump()) - part_index = 0 for tool_call in completion.choices[0].message.tool_calls or []: response.add_tool_call( llm.ToolCall( @@ -1019,24 +1009,20 @@ def execute( arguments=json.loads(tool_call.function.arguments), ) ) - part_index += 1 yield StreamEvent( type="tool_call_name", chunk=tool_call.function.name or "", - part_index=part_index, tool_call_id=tool_call.id, ) yield StreamEvent( type="tool_call_args", chunk=tool_call.function.arguments or "", - part_index=part_index, tool_call_id=tool_call.id, ) if completion.choices[0].message.content is not None: yield StreamEvent( type="text", chunk=completion.choices[0].message.content, - part_index=0, ) # Capture the reasoning token count BEFORE set_usage runs — # set_usage pops top-level keys and passes the rest through @@ -1085,8 +1071,6 @@ async def execute( ) chunks = [] tool_calls = {} - tc_part_index = {} - next_part_index = 1 async for chunk in completion: if chunk.usage: usage = chunk.usage.model_dump() @@ -1098,12 +1082,9 @@ async def execute( idx = tool_call.index if idx not in tool_calls: tool_calls[idx] = tool_call - tc_part_index[idx] = next_part_index - next_part_index += 1 yield StreamEvent( type="tool_call_name", chunk=tool_call.function.name or "", - part_index=tc_part_index[idx], tool_call_id=tool_call.id, ) else: @@ -1114,7 +1095,6 @@ async def execute( yield StreamEvent( type="tool_call_args", chunk=tool_call.function.arguments, - part_index=tc_part_index[idx], tool_call_id=tool_calls[idx].id, ) try: @@ -1122,7 +1102,7 @@ async def execute( except IndexError: content = None if content: - yield StreamEvent(type="text", chunk=content, part_index=0) + yield StreamEvent(type="text", chunk=content) if tool_calls: for value in tool_calls.values(): response.add_tool_call( @@ -1142,7 +1122,6 @@ async def execute( ) response.response_json = remove_dict_none_values(completion.model_dump()) usage = completion.usage.model_dump() - part_index = 0 for tool_call in completion.choices[0].message.tool_calls or []: response.add_tool_call( llm.ToolCall( @@ -1151,24 +1130,20 @@ async def execute( arguments=json.loads(tool_call.function.arguments), ) ) - part_index += 1 yield StreamEvent( type="tool_call_name", chunk=tool_call.function.name or "", - part_index=part_index, tool_call_id=tool_call.id, ) yield StreamEvent( type="tool_call_args", chunk=tool_call.function.arguments or "", - part_index=part_index, tool_call_id=tool_call.id, ) if completion.choices[0].message.content is not None: yield StreamEvent( type="text", chunk=completion.choices[0].message.content, - part_index=0, ) # See sync Chat.execute: capture reasoning before set_usage mutates. if usage: diff --git a/llm/models.py b/llm/models.py index ba5acc5f7..4e1101cd7 100644 --- a/llm/models.py +++ b/llm/models.py @@ -851,10 +851,24 @@ def __init__( self._key = key self._chunks: List[str] = [] # Every StreamEvent ever yielded by execute(), in order. Plain - # str yields are wrapped as StreamEvent(type="text", part_index=0) - # so this buffer is the single source of truth for replay and - # for assembling response.messages. + # str yields are wrapped as text events (with part_index resolved + # by _resolve_part_index) so this buffer is the single source of + # truth for replay and for assembling response.messages. self._stream_events: List[Any] = [] + # Auto-allocator state for resolving StreamEvent.part_index=None. + # Plugins yield events with part_index=None (the default) and + # the framework assigns concrete integers based on context: + # consecutive same-family text/reasoning events concatenate, + # tool calls group by tool_call_id, and tool_result is always + # its own part. _auto_index_max tracks the highest index seen + # (explicit or allocated); _auto_last_index / _auto_last_family + # remember the previously-resolved event so same-family runs + # share an index; _auto_tool_id_to_index maps known tool ids to + # their assigned index for parallel-tool-call grouping. + self._auto_index_max: int = -1 + self._auto_last_index: Optional[int] = None + self._auto_last_family: Optional[str] = None + self._auto_tool_id_to_index: Dict[str, int] = {} # Plugins set this when the provider reports an opaque reasoning # token count (no streamed reasoning text). _build_parts() # prepends a ReasoningPart(redacted=True, token_count=N) when @@ -884,24 +898,103 @@ def messages(self) -> List[Any]: "Overridden by Response / AsyncResponse — declared here for type checkers." raise NotImplementedError + @staticmethod + def _event_family(event_type: str) -> str: + if event_type in ("tool_call_name", "tool_call_args"): + return "tool_call" + return event_type + + def _resolve_part_index(self, event): + """Mutate event.part_index in place when the plugin left it None. + + Resolution rules: consecutive same-family text/reasoning events + share an index; tool-call events are grouped by tool_call_id; + tool_result always allocates a fresh index. Explicit indices + pass through but update the allocator's bookkeeping so future + None resolutions avoid collisions. + """ + fam = self._event_family(event.type) + + if event.part_index is not None: + if event.part_index > self._auto_index_max: + self._auto_index_max = event.part_index + if ( + event.type in ("tool_call_name", "tool_call_args") + and event.tool_call_id + ): + self._auto_tool_id_to_index[event.tool_call_id] = event.part_index + self._auto_last_index = event.part_index + self._auto_last_family = fam + return + + if event.type in ("tool_call_name", "tool_call_args"): + if event.tool_call_id: + existing = self._auto_tool_id_to_index.get(event.tool_call_id) + if existing is not None: + event.part_index = existing + self._auto_last_index = existing + self._auto_last_family = "tool_call" + return + self._auto_index_max += 1 + new_idx = self._auto_index_max + self._auto_tool_id_to_index[event.tool_call_id] = new_idx + event.part_index = new_idx + self._auto_last_index = new_idx + self._auto_last_family = "tool_call" + return + # No tool_call_id — fall back to grouping with the prior + # tool-call event if one is current. + if ( + self._auto_last_family == "tool_call" + and self._auto_last_index is not None + ): + event.part_index = self._auto_last_index + return + self._auto_index_max += 1 + new_idx = self._auto_index_max + event.part_index = new_idx + self._auto_last_index = new_idx + self._auto_last_family = "tool_call" + return + + if event.type == "tool_result": + self._auto_index_max += 1 + new_idx = self._auto_index_max + event.part_index = new_idx + self._auto_last_index = new_idx + self._auto_last_family = "tool_result" + return + + # text / reasoning: same family as previous → reuse, else new. + if self._auto_last_family == fam and self._auto_last_index is not None: + event.part_index = self._auto_last_index + return + self._auto_index_max += 1 + new_idx = self._auto_index_max + event.part_index = new_idx + self._auto_last_index = new_idx + self._auto_last_family = fam + def _process_chunk(self, chunk): """Normalize a chunk from execute() into a StreamEvent and return the text str (or None) that __iter__ should yield. Plain str yields from legacy plugins are wrapped as text events - at part_index=0. Side effects: populates self._stream_events and - self._chunks. + with an auto-allocated part_index. Side effects: populates + self._stream_events and self._chunks. """ from .parts import StreamEvent if isinstance(chunk, StreamEvent): + self._resolve_part_index(chunk) self._stream_events.append(chunk) if chunk.type == "text": self._chunks.append(chunk.chunk) return chunk.chunk return None # Legacy plain-str plugin. - event = StreamEvent(type="text", chunk=chunk, part_index=0) + event = StreamEvent(type="text", chunk=chunk) + self._resolve_part_index(event) self._stream_events.append(event) self._chunks.append(chunk) return chunk @@ -958,113 +1051,84 @@ def _build_parts(self) -> List[Any]: ) return fallback_parts - def family(t: str) -> str: - if t in ("tool_call_name", "tool_call_args"): - return "tool_call" - return t + # Group events by their (resolved) part_index, preserving the + # order in which each index was first seen. Then build one Part + # per group. This handles non-adjacent same-index events (e.g. + # text → tool_call → text where the plugin pinned both text + # bursts to part_index=0) by merging them into one Part. + groups: Dict[int, List[Any]] = {} + order: List[int] = [] + for event in self._stream_events: + pi = event.part_index + if pi not in groups: + groups[pi] = [] + order.append(pi) + groups[pi].append(event) parts: List[Any] = [] - current_index: Optional[int] = None - current_family: Optional[str] = None - text_buf: List[str] = [] - tool_name: Optional[str] = None - tool_args_buf: List[str] = [] - tool_call_id: Optional[str] = None - server_executed = False - tool_result_name: Optional[str] = None - pm_merged: Optional[Dict[str, Any]] = None - - def finalize(): - nonlocal pm_merged - if current_family is None: - return - if current_family == "text": - text = "".join(text_buf) + for pi in order: + evs = groups[pi] + fam_first = self._event_family(evs[0].type) + for e in evs: + if self._event_family(e.type) != fam_first: + raise ValueError( + f"StreamEvent type {e.type!r} is incompatible with " + f"prior type at part_index={pi}. " + "Allocate a new part_index for a different content type." + ) + + pm_merged: Optional[Dict[str, Any]] = None + for e in evs: + if e.provider_metadata: + merged = dict(pm_merged) if pm_merged else {} + for k, v in e.provider_metadata.items(): + merged[k] = v + pm_merged = merged + + if fam_first == "text": + text = "".join(e.chunk for e in evs) if text: parts.append(TextPart(text=text, provider_metadata=pm_merged)) - elif current_family == "reasoning": - text = "".join(text_buf) + elif fam_first == "reasoning": + text = "".join(e.chunk for e in evs) if text: parts.append(ReasoningPart(text=text, provider_metadata=pm_merged)) - elif current_family == "tool_call": - args_str = "".join(tool_args_buf) + elif fam_first == "tool_call": + tool_name = "".join(e.chunk for e in evs if e.type == "tool_call_name") + args_str = "".join(e.chunk for e in evs if e.type == "tool_call_args") try: arguments = json.loads(args_str) if args_str else {} except json.JSONDecodeError: arguments = {"_raw": args_str} + tool_call_id = next( + (e.tool_call_id for e in evs if e.tool_call_id), None + ) + server_executed = any(e.server_executed for e in evs) parts.append( ToolCallPart( - name=tool_name or "", + name=tool_name, arguments=arguments, tool_call_id=tool_call_id, server_executed=server_executed, provider_metadata=pm_merged, ) ) - elif current_family == "tool_result": + elif fam_first == "tool_result": + tool_result_name = next((e.tool_name for e in evs if e.tool_name), "") + tool_call_id = next( + (e.tool_call_id for e in evs if e.tool_call_id), None + ) + server_executed = any(e.server_executed for e in evs) parts.append( ToolResultPart( - name=tool_result_name or "", - output="".join(text_buf), + name=tool_result_name, + output="".join(e.chunk for e in evs), tool_call_id=tool_call_id, server_executed=server_executed, provider_metadata=pm_merged, ) ) - for event in self._stream_events: - ev_family = family(event.type) - if event.part_index != current_index: - finalize() - current_index = event.part_index - current_family = ev_family - text_buf = [] - tool_name = None - tool_args_buf = [] - tool_call_id = None - server_executed = False - tool_result_name = None - pm_merged = None - elif current_family is not None and ev_family != current_family: - raise ValueError( - f"StreamEvent type {event.type!r} is incompatible with " - f"prior type at part_index={event.part_index}. " - "Allocate a new part_index for a different content type." - ) - - if event.type == "text": - text_buf.append(event.chunk) - elif event.type == "reasoning": - text_buf.append(event.chunk) - elif event.type == "tool_call_name": - tool_name = (tool_name or "") + event.chunk - if event.tool_call_id: - tool_call_id = event.tool_call_id - if event.server_executed: - server_executed = True - elif event.type == "tool_call_args": - tool_args_buf.append(event.chunk) - if event.tool_call_id and tool_call_id is None: - tool_call_id = event.tool_call_id - if event.server_executed: - server_executed = True - elif event.type == "tool_result": - text_buf.append(event.chunk) - if event.tool_call_id and tool_call_id is None: - tool_call_id = event.tool_call_id - if event.server_executed: - server_executed = True - if event.tool_name: - tool_result_name = event.tool_name - - if event.provider_metadata: - merged = dict(pm_merged) if pm_merged else {} - for k, v in event.provider_metadata.items(): - merged[k] = v - pm_merged = merged - - finalize() - if self._reasoning_token_count: parts.insert( 0, diff --git a/llm/parts.py b/llm/parts.py index 1e23db7ed..3b7b22dd4 100644 --- a/llm/parts.py +++ b/llm/parts.py @@ -316,9 +316,13 @@ def tool_message( class StreamEvent: """A streaming event from a model response. - `part_index` groups events into parts — events sharing an index - belong to the same logical part. Mixing families (e.g. text with - tool_call_name) at the same index is a plugin bug. + `part_index` groups events into parts. When left at its default of + `None`, the framework allocates an index automatically: consecutive + same-family text/reasoning events concatenate, tool-call events + group by `tool_call_id`, and `tool_result` always starts its own + part. Pass an explicit integer only to override the default + grouping (e.g. forcing a single TextPart across non-adjacent text + bursts). `provider_metadata` carries opaque provider data (Anthropic `signature`, Gemini `thoughtSignature`, OpenAI `encrypted_content`) @@ -333,7 +337,7 @@ class StreamEvent: type: str # "text" / "reasoning" / "tool_call_name" / # "tool_call_args" / "tool_result" chunk: str - part_index: int + part_index: Optional[int] = None tool_call_id: Optional[str] = None server_executed: bool = False tool_name: Optional[str] = None diff --git a/tests/test_parts.py b/tests/test_parts.py index f0c572137..2f697ee76 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -476,6 +476,204 @@ def execute(self, prompt, stream, response, conversation): assert parts[1] == llm.TextPart(text="hi") +class TestPartIndexAutoAllocation: + """When part_index is None (the default), the framework groups + events into Parts using same-family adjacency for text/reasoning + and tool_call_id for tool calls.""" + + def test_streamevent_part_index_defaults_to_none(self): + ev = llm.StreamEvent(type="text", chunk="hi") + assert ev.part_index is None + + def test_consecutive_text_concatenates_into_one_part(self, mock_model): + events = [ + llm.StreamEvent(type="text", chunk="hello "), + llm.StreamEvent(type="text", chunk="world"), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + assert response.messages[0].parts == [llm.TextPart(text="hello world")] + + def test_text_then_reasoning_splits_into_two_parts(self, mock_model): + events = [ + llm.StreamEvent(type="text", chunk="hello"), + llm.StreamEvent(type="reasoning", chunk="thinking"), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + assert response.messages[0].parts == [ + llm.TextPart(text="hello"), + llm.ReasoningPart(text="thinking"), + ] + + def test_text_tool_call_text_produces_three_parts(self, mock_model): + events = [ + llm.StreamEvent(type="text", chunk="before"), + llm.StreamEvent( + type="tool_call_name", + chunk="search", + tool_call_id="c1", + ), + llm.StreamEvent( + type="tool_call_args", + chunk='{"q": "x"}', + tool_call_id="c1", + ), + llm.StreamEvent(type="text", chunk="after"), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + assert response.messages[0].parts == [ + llm.TextPart(text="before"), + llm.ToolCallPart(name="search", arguments={"q": "x"}, tool_call_id="c1"), + llm.TextPart(text="after"), + ] + + def test_tool_call_groups_by_tool_call_id(self, mock_model): + events = [ + llm.StreamEvent( + type="tool_call_name", + chunk="search", + tool_call_id="c1", + ), + llm.StreamEvent( + type="tool_call_args", + chunk='{"q":', + tool_call_id="c1", + ), + llm.StreamEvent( + type="tool_call_args", + chunk='"weather"}', + tool_call_id="c1", + ), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + assert response.messages[0].parts == [ + llm.ToolCallPart( + name="search", + arguments={"q": "weather"}, + tool_call_id="c1", + ) + ] + + def test_parallel_tool_calls_interleaved_by_id(self, mock_model): + # Two tool calls whose args interleave on the wire — must + # still produce two distinct ToolCallParts grouped by id. + events = [ + llm.StreamEvent(type="tool_call_name", chunk="search", tool_call_id="A"), + llm.StreamEvent(type="tool_call_name", chunk="lookup", tool_call_id="B"), + llm.StreamEvent(type="tool_call_args", chunk='{"q":"a"}', tool_call_id="A"), + llm.StreamEvent(type="tool_call_args", chunk='{"k":"b"}', tool_call_id="B"), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + parts = response.messages[0].parts + assert parts == [ + llm.ToolCallPart(name="search", arguments={"q": "a"}, tool_call_id="A"), + llm.ToolCallPart(name="lookup", arguments={"k": "b"}, tool_call_id="B"), + ] + + def test_tool_result_is_always_own_part(self, mock_model): + events = [ + llm.StreamEvent( + type="tool_call_name", + chunk="web_search", + tool_call_id="c1", + server_executed=True, + ), + llm.StreamEvent( + type="tool_call_args", + chunk='{"q":"x"}', + tool_call_id="c1", + server_executed=True, + ), + llm.StreamEvent( + type="tool_result", + chunk="results...", + tool_call_id="c1", + tool_name="web_search", + server_executed=True, + ), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + parts = response.messages[0].parts + assert parts == [ + llm.ToolCallPart( + name="web_search", + arguments={"q": "x"}, + tool_call_id="c1", + server_executed=True, + ), + llm.ToolResultPart( + name="web_search", + output="results...", + tool_call_id="c1", + server_executed=True, + ), + ] + + def test_two_reasoning_blocks_split_by_tool_call(self, mock_model): + # Some providers emit two thinking blocks separated by a tool + # call — those should yield two ReasoningParts, not one. + events = [ + llm.StreamEvent(type="reasoning", chunk="first"), + llm.StreamEvent(type="tool_call_name", chunk="t", tool_call_id="c1"), + llm.StreamEvent(type="tool_call_args", chunk="{}", tool_call_id="c1"), + llm.StreamEvent(type="reasoning", chunk="second"), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + parts = response.messages[0].parts + assert parts == [ + llm.ReasoningPart(text="first"), + llm.ToolCallPart(name="t", arguments={}, tool_call_id="c1"), + llm.ReasoningPart(text="second"), + ] + + def test_explicit_part_index_still_works(self, mock_model): + # Back-compat: plugins that pass explicit part_index should + # behave exactly as before. + events = [ + llm.StreamEvent(type="reasoning", chunk="t", part_index=0), + llm.StreamEvent(type="text", chunk="hi", part_index=1), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + assert response.messages[0].parts == [ + llm.ReasoningPart(text="t"), + llm.TextPart(text="hi"), + ] + + def test_mix_explicit_zero_and_none_for_text_concatenates(self, mock_model): + # Forcing a single TextPart across non-adjacent text bursts: + # plugin pins explicit part_index=0 on the wraparound text + # events, and the tool call in between gets None (auto). + events = [ + llm.StreamEvent(type="text", chunk="before ", part_index=0), + llm.StreamEvent(type="tool_call_name", chunk="t", tool_call_id="c1"), + llm.StreamEvent(type="tool_call_args", chunk="{}", tool_call_id="c1"), + llm.StreamEvent(type="text", chunk="after", part_index=0), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + parts = response.messages[0].parts + assert parts == [ + llm.TextPart(text="before after"), + llm.ToolCallPart(name="t", arguments={}, tool_call_id="c1"), + ] + + class TestStreamEventsLiveDuringStreaming: """Client code sees events arrive before the response is done""" From a2547d8183f082134abc88ef3a44a0781659c8ef Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 13:28:55 -0700 Subject: [PATCH 053/258] Drop token_count from ReasoningPart, use redacted marker StreamEvent ReasoningPart.token_count duplicated info already on response.token_details (reasoning_tokens), and the side-channel `response._reasoning_token_count` attribute with its set_usage ordering footgun was the wrong shape. Replaced with a clean StreamEvent.redacted=True marker that plugins yield like any other event. The framework hoists redacted reasoning Parts to the start of the assembled message so UIs render them before content, even though the opaque count typically arrives at the end of the stream. Also fix parallel tool calls emitted without tool_call_id (e.g. Gemini): a fresh tool_call_name now always allocates a new index instead of falling through to the prior tool-call group, so N parallel calls produce N distinct ToolCallParts instead of one with concatenated names and args. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/default_plugins/openai_models.py | 8 +-- llm/models.py | 60 ++++++++++--------- llm/parts.py | 21 ++++--- llm/serialization.py | 6 +- tests/test_openai_messages.py | 18 +----- tests/test_parts.py | 90 ++++++++++++++++++++++------ tests/test_serialization.py | 3 +- 7 files changed, 127 insertions(+), 79 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index fe346b65e..eddd489ae 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1024,7 +1024,7 @@ def execute( type="text", chunk=completion.choices[0].message.content, ) - # Capture the reasoning token count BEFORE set_usage runs — + # Read reasoning_tokens from usage BEFORE set_usage runs — # set_usage pops top-level keys and passes the rest through # simplify_usage_dict, which strips zero-valued entries. if usage: @@ -1032,7 +1032,7 @@ def execute( "reasoning_tokens", 0 ) if reasoning_tokens: - response._reasoning_token_count = reasoning_tokens + yield StreamEvent(type="reasoning", chunk="", redacted=True) self.set_usage(response, usage) response._prompt_json = redact_data({"messages": messages}) @@ -1145,13 +1145,13 @@ async def execute( type="text", chunk=completion.choices[0].message.content, ) - # See sync Chat.execute: capture reasoning before set_usage mutates. + # See sync Chat.execute: read reasoning before set_usage mutates. if usage: reasoning_tokens = (usage.get("completion_tokens_details") or {}).get( "reasoning_tokens", 0 ) if reasoning_tokens: - response._reasoning_token_count = reasoning_tokens + yield StreamEvent(type="reasoning", chunk="", redacted=True) self.set_usage(response, usage) response._prompt_json = redact_data({"messages": messages}) diff --git a/llm/models.py b/llm/models.py index 4e1101cd7..f4e8cc602 100644 --- a/llm/models.py +++ b/llm/models.py @@ -869,11 +869,6 @@ def __init__( self._auto_last_index: Optional[int] = None self._auto_last_family: Optional[str] = None self._auto_tool_id_to_index: Dict[str, int] = {} - # Plugins set this when the provider reports an opaque reasoning - # token count (no streamed reasoning text). _build_parts() - # prepends a ReasoningPart(redacted=True, token_count=N) when - # non-zero. - self._reasoning_token_count: int = 0 self._done = False self._tool_calls: List[ToolCall] = [] self.response_json: Optional[Dict[str, Any]] = None @@ -942,10 +937,14 @@ def _resolve_part_index(self, event): self._auto_last_index = new_idx self._auto_last_family = "tool_call" return - # No tool_call_id — fall back to grouping with the prior - # tool-call event if one is current. + # No tool_call_id — providers like Gemini omit the id on + # parallel tool calls. tool_call_args events glue onto the + # most recent tool-call index; a fresh tool_call_name + # always starts a new part (otherwise N parallel tool calls + # collapse into one with concatenated names and args). if ( - self._auto_last_family == "tool_call" + event.type == "tool_call_args" + and self._auto_last_family == "tool_call" and self._auto_last_index is not None ): event.part_index = self._auto_last_index @@ -1039,16 +1038,6 @@ def _build_parts(self) -> List[Any]: tool_call_id=tc.tool_call_id, ) ) - reasoning_token_count = getattr(self, "_reasoning_token_count", 0) - if reasoning_token_count: - fallback_parts.insert( - 0, - ReasoningPart( - text="", - redacted=True, - token_count=reasoning_token_count, - ), - ) return fallback_parts # Group events by their (resolved) part_index, preserving the @@ -1091,8 +1080,15 @@ def _build_parts(self) -> List[Any]: parts.append(TextPart(text=text, provider_metadata=pm_merged)) elif fam_first == "reasoning": text = "".join(e.chunk for e in evs) - if text: - parts.append(ReasoningPart(text=text, provider_metadata=pm_merged)) + redacted = any(e.redacted for e in evs) + if text or redacted: + parts.append( + ReasoningPart( + text=text, + redacted=redacted, + provider_metadata=pm_merged, + ) + ) elif fam_first == "tool_call": tool_name = "".join(e.chunk for e in evs if e.type == "tool_call_name") args_str = "".join(e.chunk for e in evs if e.type == "tool_call_args") @@ -1129,15 +1125,21 @@ def _build_parts(self) -> List[Any]: ) ) - if self._reasoning_token_count: - parts.insert( - 0, - ReasoningPart( - text="", - redacted=True, - token_count=self._reasoning_token_count, - ), - ) + # Hoist redacted reasoning Parts to the start of the assembled + # message. Plugins typically emit them late (when usage arrives + # in the final chunk), but UIs render reasoning before content, + # so the framework reorders. Relative order among redacted + # Parts is preserved. + redacted_parts = [ + p for p in parts if isinstance(p, ReasoningPart) and p.redacted + ] + if redacted_parts: + other_parts = [ + p + for p in parts + if not (isinstance(p, ReasoningPart) and p.redacted) + ] + parts = redacted_parts + other_parts return parts diff --git a/llm/parts.py b/llm/parts.py index 3b7b22dd4..5f5fd16eb 100644 --- a/llm/parts.py +++ b/llm/parts.py @@ -72,7 +72,6 @@ def from_dict(d: PartDict) -> "Part": return ReasoningPart( text=d["text"], redacted=d.get("redacted", False), - token_count=d.get("token_count"), provider_metadata=d.get("provider_metadata"), ) if d["type"] == "tool_call": @@ -121,22 +120,22 @@ def to_dict(self) -> TextPartDict: class ReasoningPart(Part): """Reasoning/thinking tokens from the model. - `redacted=True, text=""` represents the opaque-token-count case - (OpenAI GPT-5 series, Gemini) where the provider reports only a - count, not content. + `redacted=True, text=""` is the marker for the opaque-reasoning + case (OpenAI GPT-5 series, Gemini without `includeThoughts`) where + the provider reports that reasoning happened but withholds the + content. The actual token total lives on response.token_details + (e.g. `reasoning_tokens`); the Part only records the structural + fact that reasoning occurred. """ text: str = "" redacted: bool = False - token_count: Optional[int] = None provider_metadata: Optional[Dict[str, Any]] = None def to_dict(self) -> ReasoningPartDict: d: Dict[str, Any] = {"type": "reasoning", "text": self.text} if self.redacted: d["redacted"] = True - if self.token_count is not None: - d["token_count"] = self.token_count if self.provider_metadata: d["provider_metadata"] = self.provider_metadata return d # type: ignore[return-value] @@ -324,6 +323,13 @@ class StreamEvent: grouping (e.g. forcing a single TextPart across non-adjacent text bursts). + `redacted=True` (only meaningful on `type="reasoning"` events with + an empty `chunk`) signals that opaque reasoning happened — content + withheld by the provider, token total on response.token_details. + The framework hoists redacted reasoning Parts to the start of the + assembled message regardless of when they were emitted in the + stream, so UIs can render them before the visible content. + `provider_metadata` carries opaque provider data (Anthropic `signature`, Gemini `thoughtSignature`, OpenAI `encrypted_content`) that must be echoed back on the next request; the framework merges @@ -341,5 +347,6 @@ class StreamEvent: tool_call_id: Optional[str] = None server_executed: bool = False tool_name: Optional[str] = None + redacted: bool = False provider_metadata: Optional[Dict[str, Any]] = None message_index: int = 0 diff --git a/llm/serialization.py b/llm/serialization.py index 6ed401318..33d6400a9 100644 --- a/llm/serialization.py +++ b/llm/serialization.py @@ -81,10 +81,10 @@ class TextPartDict(TypedDict): class ReasoningPartDict(TypedDict): type: Literal["reasoning"] text: str - # Redacted reasoning: text is "" and token_count carries the opaque - # count reported by the provider (OpenAI GPT-5, Gemini thinking). + # `redacted=True` with `text=""` is the marker for opaque + # reasoning (OpenAI GPT-5, Gemini without thoughts). The token + # total lives on response usage, not on the Part. redacted: NotRequired[bool] - token_count: NotRequired[int] provider_metadata: NotRequired[Dict[str, Any]] diff --git a/tests/test_openai_messages.py b/tests/test_openai_messages.py index 67734f84b..eac0f4d33 100644 --- a/tests/test_openai_messages.py +++ b/tests/test_openai_messages.py @@ -545,19 +545,7 @@ def _text_stream_with_reasoning_usage(reasoning_tokens): class TestReasoningTokenCount: - def test_reasoning_token_count_recorded(self, httpx_mock): - httpx_mock.add_response( - method="POST", - url="https://api.openai.com/v1/chat/completions", - stream=IteratorStream(_text_stream_with_reasoning_usage(200)), - headers={"Content-Type": "text/event-stream"}, - ) - model = llm.get_model("gpt-4o-mini") - response = model.prompt("hi", key=API_KEY) - response.text() - assert response._reasoning_token_count == 200 - - def test_reasoning_part_prepended_to_messages(self, httpx_mock): + def test_redacted_reasoning_part_emitted_when_count_present(self, httpx_mock): httpx_mock.add_response( method="POST", url="https://api.openai.com/v1/chat/completions", @@ -571,7 +559,7 @@ def test_reasoning_part_prepended_to_messages(self, httpx_mock): llm.Message( role="assistant", parts=[ - llm.ReasoningPart(text="", redacted=True, token_count=150), + llm.ReasoningPart(text="", redacted=True), llm.TextPart(text="Hello"), ], ) @@ -587,8 +575,6 @@ def test_no_reasoning_part_when_zero_or_absent(self, httpx_mock): model = llm.get_model("gpt-4o-mini") response = model.prompt("hi", key=API_KEY) response.text() - # Either the attribute isn't set, or it's 0 — either way no - # redacted ReasoningPart in the assembled messages. parts = response.messages[0].parts assert not any( isinstance(p, llm.ReasoningPart) for p in parts diff --git a/tests/test_parts.py b/tests/test_parts.py index 2f697ee76..a9e9e17a4 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -27,16 +27,21 @@ def test_roundtrip_with_text(self): assert restored == part assert restored.text == "Let me think..." assert restored.redacted is False - assert restored.token_count is None def test_roundtrip_redacted(self): - part = llm.ReasoningPart(text="", redacted=True, token_count=150) + part = llm.ReasoningPart(text="", redacted=True) d = part.to_dict() assert d["redacted"] is True - assert d["token_count"] == 150 + assert "token_count" not in d restored = llm.Part.from_dict(d) assert restored == part + def test_no_token_count_field(self): + # token_count was removed: opaque token totals live on + # response.token_details, not on the Part. + with pytest.raises(TypeError): + llm.ReasoningPart(text="", redacted=True, token_count=150) + class TestToolCallPart: def test_roundtrip(self): @@ -460,20 +465,46 @@ def test_provider_metadata_merges_last_wins(self, mock_model): part = response.messages[0].parts[0] assert part.provider_metadata == {"anthropic": {"signature": "final"}} - def test_reasoning_token_count_prepends_redacted_part(self, mock_model): - # Plugin reports an opaque reasoning token count — framework - # prepends a ReasoningPart(redacted=True, token_count=N, text=""). - class CountingModel(type(mock_model)): - def execute(self, prompt, stream, response, conversation): - response._reasoning_token_count = 200 - yield llm.StreamEvent(type="text", chunk="hi", part_index=0) + def test_redacted_reasoning_event_emits_marker_part(self, mock_model): + # A reasoning StreamEvent with redacted=True yields a + # ReasoningPart(text="", redacted=True) marker — opaque token + # totals live on response.token_details, not on the Part. + events = [ + llm.StreamEvent(type="reasoning", chunk="", redacted=True), + llm.StreamEvent(type="text", chunk="hi"), + ] + mock_model.enqueue(events) + response = mock_model.prompt("x") + response.text() + parts = response.messages[0].parts + assert parts == [ + llm.ReasoningPart(text="", redacted=True), + llm.TextPart(text="hi"), + ] - m = CountingModel() - response = m.prompt("x") + def test_redacted_reasoning_hoisted_to_start_when_emitted_late( + self, mock_model + ): + # Plugins typically learn opaque reasoning happened only when + # the final usage chunk arrives, so they emit the marker last. + # The framework hoists redacted reasoning Parts to the start of + # the assembled message so UIs can render them before content. + events = [ + llm.StreamEvent(type="text", chunk="hello"), + llm.StreamEvent(type="reasoning", chunk="", redacted=True), + ] + mock_model.enqueue(events) + response = mock_model.prompt("x") response.text() parts = response.messages[0].parts - assert parts[0] == llm.ReasoningPart(text="", redacted=True, token_count=200) - assert parts[1] == llm.TextPart(text="hi") + assert parts == [ + llm.ReasoningPart(text="", redacted=True), + llm.TextPart(text="hello"), + ] + + def test_redacted_reasoning_event_default_redacted_is_false(self): + ev = llm.StreamEvent(type="reasoning", chunk="thinking") + assert ev.redacted is False class TestPartIndexAutoAllocation: @@ -639,6 +670,29 @@ def test_two_reasoning_blocks_split_by_tool_call(self, mock_model): llm.ReasoningPart(text="second"), ] + def test_parallel_tool_calls_without_id_each_get_own_part(self, mock_model): + # Gemini emits multiple functionCall parts back-to-back without + # a tool_call_id. Each tool_call_name must allocate a fresh + # part — otherwise the N tool calls collapse into one with + # concatenated names and args. + events = [ + llm.StreamEvent(type="tool_call_name", chunk="store_fact"), + llm.StreamEvent(type="tool_call_args", chunk='{"fact":"a"}'), + llm.StreamEvent(type="tool_call_name", chunk="store_fact"), + llm.StreamEvent(type="tool_call_args", chunk='{"fact":"b"}'), + llm.StreamEvent(type="tool_call_name", chunk="store_fact"), + llm.StreamEvent(type="tool_call_args", chunk='{"fact":"c"}'), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + parts = response.messages[0].parts + assert parts == [ + llm.ToolCallPart(name="store_fact", arguments={"fact": "a"}), + llm.ToolCallPart(name="store_fact", arguments={"fact": "b"}), + llm.ToolCallPart(name="store_fact", arguments={"fact": "c"}), + ] + def test_explicit_part_index_still_works(self, mock_model): # Back-compat: plugins that pass explicit part_index should # behave exactly as before. @@ -1526,13 +1580,13 @@ def test_roundtrip_preserves_tool_calls_and_results(self, mock_model): assert restored == messages def test_roundtrip_preserves_redacted_reasoning(self, mock_model): - """Redacted reasoning parts (opaque token counts) survive - round-trip — needed for accurate rendering of 'this turn used - N reasoning tokens'.""" + """The redacted=True marker on a ReasoningPart survives + round-trip — UIs use it to show that opaque reasoning happened + in this turn (the actual token count lives on response usage).""" msg = llm.Message( role="assistant", parts=[ - llm.ReasoningPart(text="", redacted=True, token_count=150), + llm.ReasoningPart(text="", redacted=True), llm.TextPart(text="result"), ], ) diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 76980771e..651608823 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -38,7 +38,6 @@ def test_reasoning_part_dict_required_keys(self): assert ReasoningPartDict.__required_keys__ == {"type", "text"} assert ReasoningPartDict.__optional_keys__ == { "redacted", - "token_count", "provider_metadata", } @@ -90,7 +89,7 @@ def test_text_part_with_provider_metadata_matches(self): self._adapter(TextPartDict).validate_python(d) def test_reasoning_part_redacted_matches(self): - d = llm.ReasoningPart(text="", redacted=True, token_count=150).to_dict() + d = llm.ReasoningPart(text="", redacted=True).to_dict() self._adapter(ReasoningPartDict).validate_python(d) def test_reasoning_part_with_signature_matches(self): From f3a0962162635b6a04ff39d12809569f300ab252 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 13:51:13 -0700 Subject: [PATCH 054/258] response.reply() auto-executes pending tool calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Zero-arg sugar: when a response made tool calls and tool_results= is not passed, reply() runs self.execute_tool_calls() and threads the results into the next turn. Pass tool_results= explicitly to skip the auto-execute path (e.g. for mutated or synthetic results). Also forwards self.prompt.tools to the next turn so the model can call the same tools again, mirroring Conversation.prompt's tools-or-self rule. AsyncResponse.reply() is now an awaitable coroutine — `await response.reply(...)` — so the auto-execute path can `await self.execute_tool_calls()` internally. This is a non-shipped API break: existing async-reply callers in the test suite updated. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/python-api.md | 22 ++- llm/models.py | 67 ++++++-- tests/test_async_parity.py | 16 +- tests/test_parts.py | 332 ++++++++++++++++++++++++++++++++++++- 4 files changed, 417 insertions(+), 20 deletions(-) diff --git a/docs/python-api.md b/docs/python-api.md index 31e9c6103..df1dd7d07 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -121,7 +121,20 @@ You can call `response.execute_tool_calls()` to execute those calls and get back tool_results = response.execute_tool_calls() # [ToolResult(name='upper', output='PANDA', tool_call_id='...')] ``` -You can use the `model.chain()` to pass the results of tool calls back to the model automatically as subsequent prompts: +To get the model's follow-up reply, call `response.reply()` — when the previous response made tool calls, `reply()` automatically executes them and feeds the results back into the next turn: +```python +follow_up = response.reply() +print(follow_up.text()) +# The word "panda" converted to uppercase is "PANDA". +``` +You can also pass an additional user prompt: `response.reply("now translate it to French")`. To use custom or already-computed tool results (e.g. results you mutated, or synthetic ones for testing) pass them explicitly with `tool_results=` and the auto-execute step is skipped: +```python +follow_up = response.reply( + "now translate it", + tool_results=[llm.ToolResult(name="upper", output="PANDA", tool_call_id="...")], +) +``` +For an automatic loop that keeps going until the model stops requesting tools, use `model.chain()` — it passes tool call results back to the model automatically as subsequent prompts: ```python chain_response = model.chain( "Convert panda to upper", @@ -703,6 +716,13 @@ async for chunk in model.chain( ): print(chunk, end="", flush=True) ``` +`response.reply()` is awaitable on async models — it `await`s `execute_tool_calls()` internally before building the next turn: +```python +response = model.prompt("Convert panda to upper", tools=[upper]) +await response.text() +follow_up = await response.reply() +print(await follow_up.text()) +``` The `before_call` and `after_call` hooks can be async functions when used with async models. (python-api-conversations)= diff --git a/llm/models.py b/llm/models.py index f4e8cc602..28f2d266e 100644 --- a/llm/models.py +++ b/llm/models.py @@ -1577,23 +1577,47 @@ def reply( prompt: Optional[str] = None, *, messages: Optional[List[Any]] = None, + tool_results: Optional[List[ToolResult]] = None, **kwargs, ) -> "Response": """Continue the conversation from this response. Builds the next turn's chain as - ``self.prompt.messages + self.messages + [user(prompt)]`` and - calls ``self.model.prompt(messages=chain, ...)``. No - Conversation object required — the Response carries everything - needed. - - If ``messages=`` is passed, its contents are appended to the - chain instead of (or in addition to) the ``prompt`` string. + ``self.prompt.messages + self.messages + [tool_message] + + [user(prompt)] + messages`` and calls + ``self.model.prompt(messages=chain, ...)``. No Conversation + object required — the Response carries everything needed. + + If this response made tool calls and ``tool_results=`` is not + passed, ``reply()`` runs ``self.execute_tool_calls()`` + automatically and threads the results into the chain. Pass an + explicit ``tool_results=`` list (e.g. results you mutated, or + synthetic ones for testing) to skip auto-execution. """ - from .parts import Message, TextPart + from .parts import Message, TextPart, ToolResultPart self._force() + if tool_results is None and self._tool_calls: + tool_results = self.execute_tool_calls() + # Forward original tools so the next turn can call them again + # (mirrors Conversation.prompt's `tools or self.tools` rule). + if "tools" not in kwargs and self.prompt.tools: + kwargs["tools"] = self.prompt.tools chain: List[Any] = list(self.prompt.messages) + list(self.messages) + if tool_results: + chain.append( + Message( + role="tool", + parts=[ + ToolResultPart( + name=tr.name, + output=tr.output, + tool_call_id=tr.tool_call_id, + ) + for tr in tool_results + ], + ) + ) if prompt: chain.append(Message(role="user", parts=[TextPart(text=prompt)])) if messages: @@ -1891,23 +1915,46 @@ class AsyncResponse(_BaseResponse): model: "AsyncModel" conversation: Optional["AsyncConversation"] = None - def reply( + async def reply( self, prompt: Optional[str] = None, *, messages: Optional[List[Any]] = None, + tool_results: Optional[List[ToolResult]] = None, **kwargs, ) -> "AsyncResponse": """Async counterpart of Response.reply(). Requires this response to have been awaited (so self.messages is available). + + Awaitable so the auto-execute path can ``await + self.execute_tool_calls()``. See ``Response.reply`` for the + ``tool_results=`` semantics. """ - from .parts import Message, TextPart + from .parts import Message, TextPart, ToolResultPart if not self._done: raise ValueError( "Response not yet awaited — call `await response` before reply()" ) + if tool_results is None and self._tool_calls: + tool_results = await self.execute_tool_calls() + if "tools" not in kwargs and self.prompt.tools: + kwargs["tools"] = self.prompt.tools chain: List[Any] = list(self.prompt.messages) + list(self.messages) + if tool_results: + chain.append( + Message( + role="tool", + parts=[ + ToolResultPart( + name=tr.name, + output=tr.output, + tool_call_id=tr.tool_call_id, + ) + for tr in tool_results + ], + ) + ) if prompt: chain.append(Message(role="user", parts=[TextPart(text=prompt)])) if messages: diff --git a/tests/test_async_parity.py b/tests/test_async_parity.py index 08d571e8d..cf8b6a570 100644 --- a/tests/test_async_parity.py +++ b/tests/test_async_parity.py @@ -72,7 +72,7 @@ async def test_async_from_dict_then_reply_continues(): payload = json.dumps(r1.to_dict()) restored = llm.AsyncResponse.from_dict(json.loads(payload)) - r2 = restored.reply("q2") + r2 = await restored.reply("q2") await r2.text() # r2 was sent the full chain including r1's output. @@ -163,7 +163,7 @@ async def _capture_async(model): await r1.text() payload1 = json.dumps(r1.to_dict()) restored = llm.AsyncResponse.from_dict(json.loads(payload1)) - r2 = restored.reply("pong") + r2 = await restored.reply("pong") await r2.text() return r2.prompt.messages @@ -315,7 +315,7 @@ async def test_async_reply_messages_kwarg_appends(): model = llm.get_async_model("echo") r1 = model.prompt("q1") await r1.text() - r2 = r1.reply(messages=[llm.user("extra")]) + r2 = await r1.reply(messages=[llm.user("extra")]) await r2.text() assert [m.role for m in r2.prompt.messages] == ["user", "assistant", "user"] assert r2.prompt.messages[-1].parts[0].text == "extra" @@ -338,9 +338,9 @@ async def test_async_full_chain_to_dict_round_trip_three_turns(): model = llm.get_async_model("echo") r1 = model.prompt("q1") await r1.text() - r2 = r1.reply("q2") + r2 = await r1.reply("q2") await r2.text() - r3 = r2.reply("q3") + r3 = await r2.reply("q3") await r3.text() payload = json.dumps(r3.to_dict()) @@ -358,7 +358,7 @@ async def test_async_full_chain_to_dict_round_trip_three_turns(): assert texts[4] == "q3" # And continuing from the restored response extends the chain. - r4 = restored.reply("q4") + r4 = await restored.reply("q4") await r4.text() assert [m.role for m in r4.prompt.messages] == [ "user", @@ -376,9 +376,9 @@ async def test_async_reply_chains_three_turns(): model = llm.get_async_model("echo") r1 = model.prompt("q1") await r1.text() - r2 = r1.reply("q2") + r2 = await r1.reply("q2") await r2.text() - r3 = r2.reply("q3") + r3 = await r2.reply("q3") await r3.text() chain = r3.prompt.messages diff --git a/tests/test_parts.py b/tests/test_parts.py index a9e9e17a4..ff038438b 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -1179,7 +1179,7 @@ async def test_async_reply(self, async_mock_model): async_mock_model.enqueue(["a2"]) r1 = async_mock_model.prompt("q1") await r1.text() - r2 = r1.reply("q2") + r2 = await r1.reply("q2") await r2.text() assert r2.prompt.messages == [ llm.user("q1"), @@ -1187,6 +1187,336 @@ async def test_async_reply(self, async_mock_model): llm.user("q2"), ] + def test_reply_with_tool_results_appends_tool_message(self, mock_model): + # The natural idiom: model.prompt(...) makes tool calls, the + # caller runs them, then reply(tool_results=...) sends the + # results back in one call. The chain should grow by a + # role="tool" message containing ToolResultParts. + from llm.parts import ( + Message, + ToolCallPart, + ToolResultPart, + ) + + # First-turn assistant message has a tool call. + first_assistant = Message( + role="assistant", + parts=[ + ToolCallPart( + name="echo", arguments={"x": 1}, tool_call_id="c1" + ) + ], + ) + + class ToolCallMock(type(mock_model)): + supports_tools = True + + def execute(self, prompt, stream, response, conversation): + # Yield the assistant turn's parts as StreamEvents so + # response.messages contains the tool call. + yield llm.StreamEvent( + type="tool_call_name", + chunk="echo", + tool_call_id="c1", + ) + yield llm.StreamEvent( + type="tool_call_args", + chunk='{"x": 1}', + tool_call_id="c1", + ) + + m = ToolCallMock() + r1 = m.prompt("call echo") + r1.text() + + tool_results = [ + llm.ToolResult(name="echo", output="ok", tool_call_id="c1") + ] + # The bug we're fixing: this previously silently dropped the + # tool_results because reply() forwards via messages= and the + # Prompt synthesis path is bypassed. + m.enqueue(["follow-up text"]) + r2 = r1.reply(tool_results=tool_results) + r2.text() + assert r2.prompt.messages == [ + llm.user("call echo"), + first_assistant, + Message( + role="tool", + parts=[ + ToolResultPart( + name="echo", output="ok", tool_call_id="c1" + ) + ], + ), + ] + + def test_reply_with_tool_results_and_prompt(self, mock_model): + from llm.parts import ( + Message, + ToolCallPart, + ToolResultPart, + ) + + class ToolCallMock(type(mock_model)): + supports_tools = True + + def execute(self, prompt, stream, response, conversation): + yield llm.StreamEvent( + type="tool_call_name", + chunk="echo", + tool_call_id="c1", + ) + yield llm.StreamEvent( + type="tool_call_args", + chunk='{"x": 1}', + tool_call_id="c1", + ) + + m = ToolCallMock() + r1 = m.prompt("call echo") + r1.text() + m.enqueue(["follow-up"]) + r2 = r1.reply( + "now summarise", + tool_results=[llm.ToolResult(name="echo", output="ok", tool_call_id="c1")], + ) + r2.text() + roles = [m.role for m in r2.prompt.messages] + assert roles == ["user", "assistant", "tool", "user"] + # tool message goes BEFORE the new user prompt. + tool_msg = r2.prompt.messages[2] + assert tool_msg.parts == [ + ToolResultPart(name="echo", output="ok", tool_call_id="c1") + ] + assert r2.prompt.messages[3] == llm.user("now summarise") + + def test_reply_auto_executes_tool_calls_when_none_passed(self, mock_model): + # Zero-arg sugar: response.reply() with tool calls present + # auto-executes them and threads results back into the chain. + from llm.parts import Message, ToolResultPart + + executed = [] + + def echo(x: int) -> str: + executed.append(x) + return f"echo:{x}" + + class ToolCallMock(type(mock_model)): + supports_tools = True + + def execute(self, prompt, stream, response, conversation): + response.add_tool_call( + llm.ToolCall( + name="echo", arguments={"x": 42}, tool_call_id="c1" + ) + ) + yield llm.StreamEvent( + type="tool_call_name", chunk="echo", tool_call_id="c1" + ) + yield llm.StreamEvent( + type="tool_call_args", + chunk='{"x": 42}', + tool_call_id="c1", + ) + + m = ToolCallMock() + r1 = m.prompt("call echo", tools=[echo]) + r1.text() + + m.enqueue(["follow-up"]) + # No tool_results passed — sugar kicks in and auto-executes. + r2 = r1.reply() + r2.text() + + assert executed == [42] + # The tool message landed in the chain. + roles = [msg.role for msg in r2.prompt.messages] + assert roles == ["user", "assistant", "tool"] + tool_msg = r2.prompt.messages[2] + assert tool_msg.parts == [ + ToolResultPart(name="echo", output="echo:42", tool_call_id="c1") + ] + + def test_reply_auto_execute_with_prompt(self, mock_model): + # reply("more text") with tool calls present also auto-executes + # so the user prompt can land after the tool results. + executed = [] + + def echo(x: int) -> str: + executed.append(x) + return "out" + + class ToolCallMock(type(mock_model)): + supports_tools = True + + def execute(self, prompt, stream, response, conversation): + response.add_tool_call( + llm.ToolCall( + name="echo", arguments={"x": 1}, tool_call_id="c1" + ) + ) + yield llm.StreamEvent( + type="tool_call_name", chunk="echo", tool_call_id="c1" + ) + yield llm.StreamEvent( + type="tool_call_args", + chunk='{"x": 1}', + tool_call_id="c1", + ) + + m = ToolCallMock() + r1 = m.prompt("call echo", tools=[echo]) + r1.text() + m.enqueue(["follow-up"]) + r2 = r1.reply("now summarise") + r2.text() + assert executed == [1] + roles = [msg.role for msg in r2.prompt.messages] + assert roles == ["user", "assistant", "tool", "user"] + + def test_reply_explicit_tool_results_skips_auto_execute(self, mock_model): + # Passing tool_results= explicitly overrides the sugar — the + # tool function does NOT run (caller already ran it / wants + # custom results). + executed = [] + + def echo(x: int) -> str: + executed.append(x) + return "should not see" + + class ToolCallMock(type(mock_model)): + supports_tools = True + + def execute(self, prompt, stream, response, conversation): + yield llm.StreamEvent( + type="tool_call_name", chunk="echo", tool_call_id="c1" + ) + yield llm.StreamEvent( + type="tool_call_args", + chunk='{"x": 1}', + tool_call_id="c1", + ) + + m = ToolCallMock() + r1 = m.prompt("call echo", tools=[echo]) + r1.text() + m.enqueue(["follow-up"]) + r2 = r1.reply( + tool_results=[llm.ToolResult(name="echo", output="custom", tool_call_id="c1")] + ) + r2.text() + assert executed == [] # echo was NOT called + tool_msg = r2.prompt.messages[2] + assert tool_msg.parts[0].output == "custom" + + def test_reply_no_tool_calls_no_tool_message(self, mock_model): + # reply() on a response without tool calls is unchanged — no + # tool message gets injected. + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + r1 = mock_model.prompt("q1") + r1.text() + r2 = r1.reply() + r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + ] + + @pytest.mark.asyncio + async def test_async_reply_auto_executes_tool_calls(self, async_mock_model): + # Async reply() is a coroutine; with tool calls present the + # zero-arg sugar awaits execute_tool_calls() internally. + from llm.parts import ToolResultPart + + executed = [] + + async def echo(x: int) -> str: + executed.append(x) + return f"echo:{x}" + + class ToolCallMock(type(async_mock_model)): + supports_tools = True + + async def execute(self, prompt, stream, response, conversation): + response.add_tool_call( + llm.ToolCall( + name="echo", arguments={"x": 7}, tool_call_id="c1" + ) + ) + yield llm.StreamEvent( + type="tool_call_name", chunk="echo", tool_call_id="c1" + ) + yield llm.StreamEvent( + type="tool_call_args", + chunk='{"x": 7}', + tool_call_id="c1", + ) + + m = ToolCallMock() + r1 = m.prompt("call echo", tools=[echo]) + await r1.text() + m.enqueue(["follow-up"]) + r2 = await r1.reply() + await r2.text() + assert executed == [7] + tool_msg = r2.prompt.messages[2] + assert tool_msg.parts == [ + ToolResultPart(name="echo", output="echo:7", tool_call_id="c1") + ] + + @pytest.mark.asyncio + async def test_async_reply_with_tool_results(self, async_mock_model): + from llm.parts import ( + Message, + ToolCallPart, + ToolResultPart, + ) + + class ToolCallMock(type(async_mock_model)): + supports_tools = True + + async def execute(self, prompt, stream, response, conversation): + yield llm.StreamEvent( + type="tool_call_name", + chunk="echo", + tool_call_id="c1", + ) + yield llm.StreamEvent( + type="tool_call_args", + chunk='{"x": 1}', + tool_call_id="c1", + ) + + m = ToolCallMock() + r1 = m.prompt("call echo") + await r1.text() + m.enqueue(["follow-up"]) + r2 = await r1.reply( + tool_results=[llm.ToolResult(name="echo", output="ok", tool_call_id="c1")] + ) + await r2.text() + assert r2.prompt.messages == [ + llm.user("call echo"), + Message( + role="assistant", + parts=[ + ToolCallPart( + name="echo", arguments={"x": 1}, tool_call_id="c1" + ) + ], + ), + Message( + role="tool", + parts=[ + ToolResultPart( + name="echo", output="ok", tool_call_id="c1" + ) + ], + ), + ] + # chain() propagates system across tool-result turns From 842ab2a93fe56ce1708af5c4d10b273b7f9f629e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 14:04:35 -0700 Subject: [PATCH 055/258] response.messages is a method, matching .text() / .json() / .tool_calls() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sync: response.messages() forces execution if not drained, so callers no longer have to remember to call .text() first. Async: `await response.messages()` awaits the force. Internal sync paths (_response_to_dict, _chain_for_tool_results, _build_full_chain, Response.reply, AsyncResponse.reply) use a new private _messages_now() helper that assumes the response is already drained, so they don't have to await on async responses. Drops the now-obsolete "accessing .messages on un-awaited AsyncResponse raises" parity test — that constraint goes away with the method form. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/python-api.md | 7 +- llm/models.py | 71 ++++++++-------- tests/test_async_parity.py | 14 +--- tests/test_openai_messages.py | 10 +-- tests/test_parts.py | 150 +++++++++++++++++++--------------- 5 files changed, 129 insertions(+), 123 deletions(-) diff --git a/docs/python-api.md b/docs/python-api.md index df1dd7d07..747d32789 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -593,16 +593,17 @@ Iterating against the response object itself (`for chunk in response`) yields on #### Inspecting the finished response -After a response completes, `response.messages` gives you the assembled list of `Message` objects returned by that response, excluding the messages from the original prompt: +`response.messages()` returns the assembled list of `Message` objects produced by the model, excluding the messages from the original prompt. Calling it forces execution if the response hasn't been drained yet, so you don't need a separate `response.text()` first: ```python response = model.prompt("What's 2+2?") -response.text() -for message in response.messages: +for message in response.messages(): for part in message.parts: print(type(part).__name__, part.to_dict()) ``` +On async models `messages()` is awaitable: `await response.messages()`. + #### Persisting a conversation A `Response` can round-trip through a plain Python dictionary via `response.to_dict()` and `llm.Response.from_dict(...)`. The dict captures the model id, the input messages that were sent, the assistant output, and any options — everything needed to continue the conversation later. diff --git a/llm/models.py b/llm/models.py index 28f2d266e..8442e72a4 100644 --- a/llm/models.py +++ b/llm/models.py @@ -535,7 +535,7 @@ def _build_full_chain( # under the invariant, so use the last response only and then # append that response's structured output. chain.extend(last.prompt.messages) - chain.extend(last.messages) + chain.extend(last._messages_now()) # Append the new turn's input if tool_results: @@ -888,11 +888,28 @@ def __init__( if self.prompt.tools and not self.model.supports_tools: raise ValueError(f"{self.model} does not support tools") - @property def messages(self) -> List[Any]: "Overridden by Response / AsyncResponse — declared here for type checkers." raise NotImplementedError + def _messages_now(self) -> List[Any]: + """Assemble messages assuming the response is already drained. + + Public ``messages()`` forces / awaits first, then delegates here. + Internal sync paths (``_response_to_dict``, + ``_chain_for_tool_results``) call this directly so they don't + have to await on async responses. + """ + from .parts import Message + + loaded = getattr(self, "_loaded_messages", None) + if loaded is not None: + return list(loaded) + parts = self._build_parts() + if not parts: + return [] + return [Message(role="assistant", parts=parts)] + @staticmethod def _event_family(event_type: str) -> str: if event_type in ("tool_call_name", "tool_call_args"): @@ -1135,9 +1152,7 @@ def _build_parts(self) -> List[Any]: ] if redacted_parts: other_parts = [ - p - for p in parts - if not (isinstance(p, ReasoningPart) and p.redacted) + p for p in parts if not (isinstance(p, ReasoningPart) and p.redacted) ] parts = redacted_parts + other_parts @@ -1484,7 +1499,7 @@ def _response_to_dict(response: "_BaseResponse") -> ResponseDict: "prompt": { "messages": [m.to_dict() for m in response.prompt.messages], }, - "messages": [m.to_dict() for m in response.messages], + "messages": [m.to_dict() for m in response._messages_now()], } if options: payload["prompt"]["options"] = options @@ -1603,7 +1618,7 @@ def reply( # (mirrors Conversation.prompt's `tools or self.tools` rule). if "tools" not in kwargs and self.prompt.tools: kwargs["tools"] = self.prompt.tools - chain: List[Any] = list(self.prompt.messages) + list(self.messages) + chain: List[Any] = list(self.prompt.messages) + list(self._messages_now()) if tool_results: chain.append( Message( @@ -1880,7 +1895,6 @@ def stream_events(self): self._done = True self._on_done() - @property def messages(self) -> List[Any]: """List of Message objects produced by this response. @@ -1888,19 +1902,15 @@ def messages(self) -> List[Any]: possible for providers that emit multi-message responses during server-side tool execution. + Forces execution if the response has not yet been drained, so + ``response.messages()`` is safe to call without a prior + ``response.text()`` / iteration. + Responses rehydrated via ``Response.from_dict`` short-circuit and return the stored messages directly. """ - from .parts import Message - - loaded = getattr(self, "_loaded_messages", None) - if loaded is not None: - return list(loaded) self._force() - parts = self._build_parts() - if not parts: - return [] - return [Message(role="assistant", parts=parts)] + return self._messages_now() def __repr__(self): text = "... not yet done ..." @@ -1940,7 +1950,7 @@ async def reply( tool_results = await self.execute_tool_calls() if "tools" not in kwargs and self.prompt.tools: kwargs["tools"] = self.prompt.tools - chain: List[Any] = list(self.prompt.messages) + list(self.messages) + chain: List[Any] = list(self.prompt.messages) + list(self._messages_now()) if tool_results: chain.append( Message( @@ -2242,26 +2252,17 @@ async def astream_events(self): finally: pass - @property - def messages(self) -> List[Any]: + async def messages(self) -> List[Any]: """List of Message objects produced by this response. - Raises ValueError if the response has not yet been awaited — - assembly depends on the full event stream. Responses rehydrated - via ``AsyncResponse.from_dict`` short-circuit and return the + Awaits ``self._force()`` so ``await response.messages()`` is + safe to call without first awaiting ``response.text()`` or + iterating the stream. Responses rehydrated via + ``AsyncResponse.from_dict`` short-circuit and return the stored messages. """ - from .parts import Message - - loaded = getattr(self, "_loaded_messages", None) - if loaded is not None: - return list(loaded) - if not self._done: - raise ValueError("Response not yet awaited — use 'await response' first") - parts = self._build_parts() - if not parts: - return [] - return [Message(role="assistant", parts=parts)] + await self._force() + return self._messages_now() async def _force(self): if not self._done: @@ -2407,7 +2408,7 @@ def _chain_for_tool_results(prior_response, tool_results, attachments) -> List[A ) chain: List[Any] = list(prior_response.prompt.messages) + list( - prior_response.messages + prior_response._messages_now() ) if tool_results: chain.append( diff --git a/tests/test_async_parity.py b/tests/test_async_parity.py index cf8b6a570..68ea1e3ac 100644 --- a/tests/test_async_parity.py +++ b/tests/test_async_parity.py @@ -56,7 +56,7 @@ async def test_async_from_dict_rehydrates(): # text_or_raise should match (same text as original) assert restored.text_or_raise() == r.text_or_raise() # messages structure preserved - assert restored.messages == r.messages + assert await restored.messages() == await r.messages() # prompt.messages (the chain that was sent) preserved assert restored.prompt.messages == r.prompt.messages @@ -108,7 +108,7 @@ async def test_async_from_row_response_messages_synthesized(tmp_path): assert rehydrated._stream_events == [] # response.messages falls back to _chunks — must not be empty. - msgs = rehydrated.messages + msgs = await rehydrated.messages() assert len(msgs) == 1 assert msgs[0].role == "assistant" assert isinstance(msgs[0].parts[0], llm.TextPart) @@ -321,16 +321,6 @@ async def test_async_reply_messages_kwarg_appends(): assert r2.prompt.messages[-1].parts[0].text == "extra" -@pytest.mark.asyncio -async def test_async_messages_requires_await_before_to_dict(): - """Parity: accessing response.messages on an un-awaited - AsyncResponse raises, matching to_dict's guard.""" - model = llm.get_async_model("echo") - r = model.prompt("hi") - with pytest.raises(ValueError): - r.messages - - @pytest.mark.asyncio async def test_async_full_chain_to_dict_round_trip_three_turns(): """Serialize on turn 3 — chain must include q1, a1, q2, a2, q3 on diff --git a/tests/test_openai_messages.py b/tests/test_openai_messages.py index eac0f4d33..e9341eab5 100644 --- a/tests/test_openai_messages.py +++ b/tests/test_openai_messages.py @@ -425,7 +425,7 @@ def test_text_stream_messages_assembled(self, httpx_mock): model = llm.get_model("gpt-4o-mini") response = model.prompt("hi", key=API_KEY) response.text() - assert response.messages == [ + assert response.messages() == [ llm.Message(role="assistant", parts=[llm.TextPart(text="Hello")]) ] @@ -495,7 +495,7 @@ def get_weather(c: int) -> str: response = model.prompt("q", tools=[get_weather], key=API_KEY) response.text() # After streaming, messages has both a TextPart and a ToolCallPart. - parts = response.messages[0].parts + parts = response.messages()[0].parts assert any(isinstance(p, llm.TextPart) for p in parts) assert any(isinstance(p, llm.ToolCallPart) for p in parts) text_part = next(p for p in parts if isinstance(p, llm.TextPart)) @@ -555,7 +555,7 @@ def test_redacted_reasoning_part_emitted_when_count_present(self, httpx_mock): model = llm.get_model("gpt-4o-mini") response = model.prompt("hi", key=API_KEY) response.text() - assert response.messages == [ + assert response.messages() == [ llm.Message( role="assistant", parts=[ @@ -575,7 +575,7 @@ def test_no_reasoning_part_when_zero_or_absent(self, httpx_mock): model = llm.get_model("gpt-4o-mini") response = model.prompt("hi", key=API_KEY) response.text() - parts = response.messages[0].parts + parts = response.messages()[0].parts assert not any( isinstance(p, llm.ReasoningPart) for p in parts ), "should not add a redacted reasoning part when count=0" @@ -606,6 +606,6 @@ def test_non_streaming_text_yields_single_event(self, httpx_mock): response = model.prompt("hi", key=API_KEY, stream=False) events = list(response.stream_events()) assert events == [llm.StreamEvent(type="text", chunk="Hello", part_index=0)] - assert response.messages == [ + assert response.messages() == [ llm.Message(role="assistant", parts=[llm.TextPart(text="Hello")]) ] diff --git a/tests/test_parts.py b/tests/test_parts.py index ff038438b..59dafb093 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -314,7 +314,7 @@ def test_response_messages_is_single_assistant_text(self, mock_model): mock_model.enqueue(["hello"]) response = mock_model.prompt("hi") response.text() - messages = response.messages + messages = response.messages() assert messages == [ llm.Message(role="assistant", parts=[llm.TextPart(text="hello")]) ] @@ -323,7 +323,7 @@ def test_empty_response_has_empty_messages(self, mock_model): mock_model.enqueue([]) response = mock_model.prompt("hi") response.text() - assert response.messages == [] + assert response.messages() == [] class TestStreamEventsFromStreamEventPlugin: @@ -359,7 +359,7 @@ def test_messages_assembles_reasoning_then_text(self, mock_model): mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() - assert response.messages == [ + assert response.messages() == [ llm.Message( role="assistant", parts=[ @@ -394,7 +394,7 @@ def test_tool_call_name_and_args_merge(self, mock_model): mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() - msgs = response.messages + msgs = response.messages() assert len(msgs) == 1 parts = msgs[0].parts assert parts == [ @@ -424,7 +424,7 @@ def test_tool_call_args_unparseable_json_falls_back(self, mock_model): mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() - part = response.messages[0].parts[0] + part = response.messages()[0].parts[0] assert part.name == "t" assert part.arguments == {"_raw": "not json"} @@ -442,7 +442,7 @@ def test_family_mismatch_at_same_part_index_raises(self, mock_model): response = mock_model.prompt("hi") response.text() with pytest.raises(ValueError, match="part_index"): - response.messages # noqa: B018 + response.messages() # noqa: B018 def test_provider_metadata_merges_last_wins(self, mock_model): events = [ @@ -462,7 +462,7 @@ def test_provider_metadata_merges_last_wins(self, mock_model): mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() - part = response.messages[0].parts[0] + part = response.messages()[0].parts[0] assert part.provider_metadata == {"anthropic": {"signature": "final"}} def test_redacted_reasoning_event_emits_marker_part(self, mock_model): @@ -476,15 +476,13 @@ def test_redacted_reasoning_event_emits_marker_part(self, mock_model): mock_model.enqueue(events) response = mock_model.prompt("x") response.text() - parts = response.messages[0].parts + parts = response.messages()[0].parts assert parts == [ llm.ReasoningPart(text="", redacted=True), llm.TextPart(text="hi"), ] - def test_redacted_reasoning_hoisted_to_start_when_emitted_late( - self, mock_model - ): + def test_redacted_reasoning_hoisted_to_start_when_emitted_late(self, mock_model): # Plugins typically learn opaque reasoning happened only when # the final usage chunk arrives, so they emit the marker last. # The framework hoists redacted reasoning Parts to the start of @@ -496,7 +494,7 @@ def test_redacted_reasoning_hoisted_to_start_when_emitted_late( mock_model.enqueue(events) response = mock_model.prompt("x") response.text() - parts = response.messages[0].parts + parts = response.messages()[0].parts assert parts == [ llm.ReasoningPart(text="", redacted=True), llm.TextPart(text="hello"), @@ -524,7 +522,7 @@ def test_consecutive_text_concatenates_into_one_part(self, mock_model): mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() - assert response.messages[0].parts == [llm.TextPart(text="hello world")] + assert response.messages()[0].parts == [llm.TextPart(text="hello world")] def test_text_then_reasoning_splits_into_two_parts(self, mock_model): events = [ @@ -534,7 +532,7 @@ def test_text_then_reasoning_splits_into_two_parts(self, mock_model): mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() - assert response.messages[0].parts == [ + assert response.messages()[0].parts == [ llm.TextPart(text="hello"), llm.ReasoningPart(text="thinking"), ] @@ -557,7 +555,7 @@ def test_text_tool_call_text_produces_three_parts(self, mock_model): mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() - assert response.messages[0].parts == [ + assert response.messages()[0].parts == [ llm.TextPart(text="before"), llm.ToolCallPart(name="search", arguments={"q": "x"}, tool_call_id="c1"), llm.TextPart(text="after"), @@ -584,7 +582,7 @@ def test_tool_call_groups_by_tool_call_id(self, mock_model): mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() - assert response.messages[0].parts == [ + assert response.messages()[0].parts == [ llm.ToolCallPart( name="search", arguments={"q": "weather"}, @@ -604,7 +602,7 @@ def test_parallel_tool_calls_interleaved_by_id(self, mock_model): mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() - parts = response.messages[0].parts + parts = response.messages()[0].parts assert parts == [ llm.ToolCallPart(name="search", arguments={"q": "a"}, tool_call_id="A"), llm.ToolCallPart(name="lookup", arguments={"k": "b"}, tool_call_id="B"), @@ -635,7 +633,7 @@ def test_tool_result_is_always_own_part(self, mock_model): mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() - parts = response.messages[0].parts + parts = response.messages()[0].parts assert parts == [ llm.ToolCallPart( name="web_search", @@ -663,7 +661,7 @@ def test_two_reasoning_blocks_split_by_tool_call(self, mock_model): mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() - parts = response.messages[0].parts + parts = response.messages()[0].parts assert parts == [ llm.ReasoningPart(text="first"), llm.ToolCallPart(name="t", arguments={}, tool_call_id="c1"), @@ -686,7 +684,7 @@ def test_parallel_tool_calls_without_id_each_get_own_part(self, mock_model): mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() - parts = response.messages[0].parts + parts = response.messages()[0].parts assert parts == [ llm.ToolCallPart(name="store_fact", arguments={"fact": "a"}), llm.ToolCallPart(name="store_fact", arguments={"fact": "b"}), @@ -703,7 +701,7 @@ def test_explicit_part_index_still_works(self, mock_model): mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() - assert response.messages[0].parts == [ + assert response.messages()[0].parts == [ llm.ReasoningPart(text="t"), llm.TextPart(text="hi"), ] @@ -721,7 +719,7 @@ def test_mix_explicit_zero_and_none_for_text_concatenates(self, mock_model): mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() - parts = response.messages[0].parts + parts = response.messages()[0].parts assert parts == [ llm.TextPart(text="before after"), llm.ToolCallPart(name="t", arguments={}, tool_call_id="c1"), @@ -796,18 +794,54 @@ async def test_async_iter_yields_only_text(self, async_mock_model): assert chunks == ["hi"] @pytest.mark.asyncio - async def test_async_messages_requires_await(self, async_mock_model): + async def test_async_messages_after_await(self, async_mock_model): async_mock_model.enqueue(["hi"]) response = async_mock_model.prompt("x") - with pytest.raises(ValueError): - response.messages # noqa: B018 + await response.text() + assert await response.messages() == [ + llm.Message(role="assistant", parts=[llm.TextPart(text="hi")]) + ] + + +class TestMessagesIsCallable: + """response.messages() is a method (matching .text(), .json(), + .tool_calls()) — invocation forces execution if not yet done. + """ + + def test_sync_messages_is_callable_and_returns_list(self, mock_model): + mock_model.enqueue(["hi"]) + response = mock_model.prompt("x") + # No prior .text() or iteration — calling messages() forces + # execution and returns the assembled list. + assert response.messages() == [ + llm.Message(role="assistant", parts=[llm.TextPart(text="hi")]) + ] + + def test_sync_messages_after_text_returns_same_list(self, mock_model): + mock_model.enqueue(["hi"]) + response = mock_model.prompt("x") + response.text() + assert response.messages() == [ + llm.Message(role="assistant", parts=[llm.TextPart(text="hi")]) + ] @pytest.mark.asyncio - async def test_async_messages_after_await(self, async_mock_model): + async def test_async_messages_is_awaitable(self, async_mock_model): + async_mock_model.enqueue(["hi"]) + response = async_mock_model.prompt("x") + # No prior await — `await response.messages()` forces it. + result = await response.messages() + assert result == [ + llm.Message(role="assistant", parts=[llm.TextPart(text="hi")]) + ] + + @pytest.mark.asyncio + async def test_async_messages_after_text_returns_same_list(self, async_mock_model): async_mock_model.enqueue(["hi"]) response = async_mock_model.prompt("x") await response.text() - assert response.messages == [ + result = await response.messages() + assert result == [ llm.Message(role="assistant", parts=[llm.TextPart(text="hi")]) ] @@ -1051,7 +1085,7 @@ async def test_async_conversation_full_chain(self, async_mock_model): class TestSqliteRehydrateMessages: - """After Response.from_row, response.messages must still yield the + """After Response.from_row, response.messages() must still yield the assistant turn as a TextPart (+ any tool calls). Otherwise Conversation.prompt builds a broken chain for `llm -c`. """ @@ -1074,10 +1108,10 @@ def test_from_row_response_messages_synthesized_from_chunks( row = next(db["responses"].rows) rehydrated = llm.Response.from_row(db, row) # _stream_events is empty (SQLite doesn't persist those), but - # _chunks carries the text. response.messages must fall back + # _chunks carries the text. response.messages() must fall back # to synthesizing a TextPart. assert rehydrated._stream_events == [] - assert rehydrated.messages == [ + assert rehydrated.messages() == [ llm.Message(role="assistant", parts=[llm.TextPart(text="answer text")]) ] @@ -1201,11 +1235,7 @@ def test_reply_with_tool_results_appends_tool_message(self, mock_model): # First-turn assistant message has a tool call. first_assistant = Message( role="assistant", - parts=[ - ToolCallPart( - name="echo", arguments={"x": 1}, tool_call_id="c1" - ) - ], + parts=[ToolCallPart(name="echo", arguments={"x": 1}, tool_call_id="c1")], ) class ToolCallMock(type(mock_model)): @@ -1213,7 +1243,7 @@ class ToolCallMock(type(mock_model)): def execute(self, prompt, stream, response, conversation): # Yield the assistant turn's parts as StreamEvents so - # response.messages contains the tool call. + # response.messages() contains the tool call. yield llm.StreamEvent( type="tool_call_name", chunk="echo", @@ -1229,9 +1259,7 @@ def execute(self, prompt, stream, response, conversation): r1 = m.prompt("call echo") r1.text() - tool_results = [ - llm.ToolResult(name="echo", output="ok", tool_call_id="c1") - ] + tool_results = [llm.ToolResult(name="echo", output="ok", tool_call_id="c1")] # The bug we're fixing: this previously silently dropped the # tool_results because reply() forwards via messages= and the # Prompt synthesis path is bypassed. @@ -1243,11 +1271,7 @@ def execute(self, prompt, stream, response, conversation): first_assistant, Message( role="tool", - parts=[ - ToolResultPart( - name="echo", output="ok", tool_call_id="c1" - ) - ], + parts=[ToolResultPart(name="echo", output="ok", tool_call_id="c1")], ), ] @@ -1307,9 +1331,7 @@ class ToolCallMock(type(mock_model)): def execute(self, prompt, stream, response, conversation): response.add_tool_call( - llm.ToolCall( - name="echo", arguments={"x": 42}, tool_call_id="c1" - ) + llm.ToolCall(name="echo", arguments={"x": 42}, tool_call_id="c1") ) yield llm.StreamEvent( type="tool_call_name", chunk="echo", tool_call_id="c1" @@ -1352,9 +1374,7 @@ class ToolCallMock(type(mock_model)): def execute(self, prompt, stream, response, conversation): response.add_tool_call( - llm.ToolCall( - name="echo", arguments={"x": 1}, tool_call_id="c1" - ) + llm.ToolCall(name="echo", arguments={"x": 1}, tool_call_id="c1") ) yield llm.StreamEvent( type="tool_call_name", chunk="echo", tool_call_id="c1" @@ -1403,7 +1423,9 @@ def execute(self, prompt, stream, response, conversation): r1.text() m.enqueue(["follow-up"]) r2 = r1.reply( - tool_results=[llm.ToolResult(name="echo", output="custom", tool_call_id="c1")] + tool_results=[ + llm.ToolResult(name="echo", output="custom", tool_call_id="c1") + ] ) r2.text() assert executed == [] # echo was NOT called @@ -1441,9 +1463,7 @@ class ToolCallMock(type(async_mock_model)): async def execute(self, prompt, stream, response, conversation): response.add_tool_call( - llm.ToolCall( - name="echo", arguments={"x": 7}, tool_call_id="c1" - ) + llm.ToolCall(name="echo", arguments={"x": 7}, tool_call_id="c1") ) yield llm.StreamEvent( type="tool_call_name", chunk="echo", tool_call_id="c1" @@ -1502,18 +1522,12 @@ async def execute(self, prompt, stream, response, conversation): Message( role="assistant", parts=[ - ToolCallPart( - name="echo", arguments={"x": 1}, tool_call_id="c1" - ) + ToolCallPart(name="echo", arguments={"x": 1}, tool_call_id="c1") ], ), Message( role="tool", - parts=[ - ToolResultPart( - name="echo", output="ok", tool_call_id="c1" - ) - ], + parts=[ToolResultPart(name="echo", output="ok", tool_call_id="c1")], ), ] @@ -1715,7 +1729,7 @@ def test_from_dict_rehydrates_with_messages(self, mock_model): restored = llm.Response.from_dict(json.loads(payload)) assert restored._done assert restored.text() == "hello" - assert restored.messages == [llm.assistant("hello")] + assert restored.messages() == [llm.assistant("hello")] assert restored.prompt.messages == [llm.user("hi")] def test_from_dict_then_reply_continues_conversation(self, mock_model): @@ -1755,7 +1769,7 @@ def test_to_dict_preserves_reasoning_and_signatures(self, mock_model): payload = json.dumps(r.to_dict()) restored = llm.Response.from_dict(json.loads(payload)) - msgs = restored.messages + msgs = restored.messages() assert msgs[0].role == "assistant" assert isinstance(msgs[0].parts[0], llm.ReasoningPart) assert msgs[0].parts[0].text == "thinking..." @@ -1854,11 +1868,11 @@ def test_response_messages_json_roundtrip(self, mock_model): r.text() # Serialize via Message.to_dict / json.dumps - payload = json.dumps([m.to_dict() for m in r.messages]) + payload = json.dumps([m.to_dict() for m in r.messages()]) # Deserialize — no LLM state needed beyond the types. restored = [llm.Message.from_dict(d) for d in json.loads(payload)] - assert restored == r.messages + assert restored == r.messages() def test_rebuilt_messages_reach_plugin_via_prompt(self, mock_model): """Round-trip: serialize messages from turn 1, re-inflate, send @@ -1870,7 +1884,7 @@ def test_rebuilt_messages_reach_plugin_via_prompt(self, mock_model): # Persist everything the client cares about. history = [llm.user("turn 1 question").to_dict()] + [ - m.to_dict() for m in r1.messages + m.to_dict() for m in r1.messages() ] payload = json.dumps(history) @@ -1882,7 +1896,7 @@ def test_rebuilt_messages_reach_plugin_via_prompt(self, mock_model): # The plugin saw the full structured history on prompt.messages. assert r2.prompt.messages == rebuilt + [llm.user("turn 2 question")] - assert r2.messages == [llm.assistant("turn 2 answer")] + assert r2.messages() == [llm.assistant("turn 2 answer")] def test_roundtrip_preserves_tool_calls_and_results(self, mock_model): """Assistant messages with tool calls + subsequent tool role From b97a6902f5140af28330dd7f3bfcd42a6e1c48f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 14:27:22 -0700 Subject: [PATCH 056/258] Persist visible reasoning to logs and render in markdown Adds a `reasoning` column to the responses table (migration m022) populated from concatenated visible-reasoning text in the assembled message. `llm logs --md` renders it under a `## Reasoning` heading above the response when present. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/logging.md | 3 +- llm/cli.py | 35 +++++++++++++------ llm/migrations.py | 8 +++++ llm/models.py | 11 ++++++ tests/test_chat.py | 12 +++++-- tests/test_llm_logs.py | 76 ++++++++++++++++++++++++++++++++++++++++-- tests/test_migrate.py | 1 + 7 files changed, 131 insertions(+), 15 deletions(-) diff --git a/docs/logging.md b/docs/logging.md index d1a46dcd3..75d6be0f9 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -333,7 +333,8 @@ CREATE TABLE "responses" ( [output_tokens] INTEGER, [token_details] TEXT, [schema_id] TEXT REFERENCES [schemas]([id]), - [resolved_model] TEXT + [resolved_model] TEXT, + [reasoning] TEXT ); CREATE VIRTUAL TABLE [responses_fts] USING FTS5 ( [prompt], diff --git a/llm/cli.py b/llm/cli.py index f5cdd586d..a53b54063 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -1468,6 +1468,7 @@ def logs_turn_off(): responses.prompt_json, responses.options_json, responses.response, + responses.reasoning, responses.response_json, responses.conversation_id, responses.duration_ms, @@ -1766,14 +1767,16 @@ def logs_list( if any_tools: # Any response that involved at least one tool result - where_bits.append(""" + where_bits.append( + """ exists ( select 1 from tool_results where tool_results.response_id = responses.id ) - """) + """ + ) if tools: tools_by_name = get_tools() # Filter responses by tools (must have ALL of the named tools, including plugin) @@ -1784,7 +1787,8 @@ def logs_list( except KeyError: raise click.ClickException(f"Unknown tool: {tool_name}") - tool_clauses.append(f""" + tool_clauses.append( + f""" exists ( select 1 from tool_results @@ -1793,7 +1797,8 @@ def logs_list( and tools.name = :tool{i} and tools.plugin = :plugin{i} ) - """) + """ + ) sql_params[f"tool{i}"] = tool_name sql_params[f"plugin{i}"] = plugin_name @@ -2217,6 +2222,8 @@ def _display_fragments(fragments, title): response = "```json\n{}\n```".format(json.dumps(parsed, indent=2)) except ValueError: pass + if row.get("reasoning"): + click.echo("\n## Reasoning\n\n{}".format(row["reasoning"])) click.echo("\n## Response\n") if row["tool_calls"]: click.echo("### Tool calls\n") @@ -2523,7 +2530,9 @@ def schemas_list(path, database, queries, full, json_, nl): on responses.schema_id = schemas.id {} group by responses.schema_id order by recently_used - """.format(where_sql) + """.format( + where_sql + ) rows = db.query(sql, params) if json_ or nl: @@ -2862,11 +2871,13 @@ def fragments_list(queries, aliases, json_): param_count += 1 p = f"p{param_count}" params[p] = q - where_bits.append(f""" + where_bits.append( + f""" (fragments.hash = :{p} or fragment_aliases.alias = :{p} or fragments.source like '%' || :{p} || '%' or fragments.content like '%' || :{p} || '%') - """) + """ + ) where = "\n and\n ".join(where_bits) if where: where = " where " + where @@ -2888,7 +2899,9 @@ def fragments_list(queries, aliases, json_): group by fragments.id, fragments.hash, fragments.content, fragments.datetime_utc, fragments.source order by fragments.datetime_utc - """.format(where=where) + """.format( + where=where + ) results = list(db.query(sql, params)) for result in results: result["aliases"] = json.loads(result["aliases"]) @@ -3578,7 +3591,8 @@ def embed_db_collections(database, json_): db = sqlite_utils.Database(str(database)) if not db["collections"].exists(): raise click.ClickException("No collections table found in {}".format(database)) - rows = db.query(""" + rows = db.query( + """ select collections.name, collections.model, @@ -3588,7 +3602,8 @@ def embed_db_collections(database, json_): on collections.id = embeddings.collection_id group by collections.name, collections.model - """) + """ + ) if json_: click.echo(json.dumps(list(rows), indent=4)) else: diff --git a/llm/migrations.py b/llm/migrations.py index f2ca04651..985aaa62f 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -418,3 +418,11 @@ def m020_tool_results_attachments(db): @migration def m021_tool_results_exception(db): db["tool_results"].add_column("exception", str) + + +@migration +def m022_response_reasoning(db): + # Concatenated visible reasoning text emitted during the response. + # NULL/empty when no reasoning was emitted or when the provider + # only reported an opaque token count (the redacted-marker case). + db["responses"].add_column("reasoning", str) diff --git a/llm/models.py b/llm/models.py index 8442e72a4..2ca64846a 100644 --- a/llm/models.py +++ b/llm/models.py @@ -1345,6 +1345,16 @@ def log_to_db(self, db): response_text = self.text_or_raise() replacements[f"r:{response_id}"] = response_text + # Concatenate visible reasoning text from the assembled + # ReasoningPart entries; redacted markers contribute nothing. + from .parts import ReasoningPart + + reasoning_text = "".join( + p.text + for m in self._messages_now() + for p in m.parts + if isinstance(p, ReasoningPart) and p.text + ) json_data = self.json() response = { @@ -1359,6 +1369,7 @@ def log_to_db(self, db): if value is not None }, "response": response_text, + "reasoning": reasoning_text or None, "response_json": condense_json(json_data, replacements), "conversation_id": conversation.id, "duration_ms": self.duration_ms(), diff --git a/tests/test_chat.py b/tests/test_chat.py index 4563f301f..e455bd2f0 100644 --- a/tests/test_chat.py +++ b/tests/test_chat.py @@ -60,6 +60,7 @@ def test_chat_basic(mock_model, logs_db): "output_tokens": 1, "token_details": None, "schema_id": None, + "reasoning": None, }, { "id": ANY, @@ -78,6 +79,7 @@ def test_chat_basic(mock_model, logs_db): "output_tokens": 1, "token_details": None, "schema_id": None, + "reasoning": None, }, ] # Now continue that conversation @@ -126,6 +128,7 @@ def test_chat_basic(mock_model, logs_db): "output_tokens": 1, "token_details": None, "schema_id": None, + "reasoning": None, } ] @@ -170,6 +173,7 @@ def test_chat_system(mock_model, logs_db): "output_tokens": 1, "token_details": None, "schema_id": None, + "reasoning": None, } ] @@ -213,6 +217,7 @@ def test_chat_options(mock_model, logs_db, user_path): "output_tokens": 1, "token_details": None, "schema_id": None, + "reasoning": None, }, { "id": ANY, @@ -231,6 +236,7 @@ def test_chat_options(mock_model, logs_db, user_path): "output_tokens": 1, "token_details": None, "schema_id": None, + "reasoning": None, }, ] @@ -308,11 +314,13 @@ def test_llm_chat_creates_log_database(tmpdir, monkeypatch, custom_database_path @pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows") def test_chat_tools(logs_db): runner = CliRunner() - functions = textwrap.dedent(""" + functions = textwrap.dedent( + """ def upper(text: str) -> str: "Convert text to upper case" return text.upper() - """) + """ + ) result = runner.invoke( llm.cli.cli, ["chat", "-m", "echo", "--functions", functions], diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index 0af16b269..3a37e117a 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -934,10 +934,12 @@ def test_expand_fragment_markdown(fragments_fixture): def test_logs_tools(logs_db): runner = CliRunner() - code = textwrap.dedent(""" + code = textwrap.dedent( + """ def demo(): return "one\\ntwo\\nthree" - """) + """ + ) result1 = runner.invoke( cli, [ @@ -1009,3 +1011,73 @@ def test_logs_resolved_model(logs_db, mock_model, async_mock_model, async_): # And the rendered logs result3 = runner.invoke(cli, ["logs"]) assert "Model: **mock** (resolved: **resolved-mock**)" in result3.output + + +# ---- Reasoning persistence and markdown rendering ----------------- + + +def test_log_to_db_persists_visible_reasoning(logs_db, mock_model): + """A response that streams reasoning events should round-trip the + visible reasoning text via the new responses.reasoning column.""" + import llm + + mock_model.enqueue( + [ + llm.StreamEvent(type="reasoning", chunk="thinking "), + llm.StreamEvent(type="reasoning", chunk="hard"), + llm.StreamEvent(type="text", chunk="hello"), + ] + ) + response = mock_model.prompt("hi") + response.text() + response.log_to_db(logs_db) + + row = next(logs_db["responses"].rows) + assert row["response"] == "hello" + assert row["reasoning"] == "thinking hard" + + +def test_log_to_db_persists_empty_reasoning_when_absent(logs_db, mock_model): + """No reasoning emitted → empty/null reasoning column, never raises.""" + mock_model.enqueue(["just text"]) + response = mock_model.prompt("hi") + response.text() + response.log_to_db(logs_db) + row = next(logs_db["responses"].rows) + assert not row.get("reasoning") + + +def test_logs_markdown_renders_reasoning_heading(user_path): + """When a row has reasoning text, `llm logs` renders a `## Reasoning` + heading between System and Response.""" + log_path = str(user_path / "logs_with_reasoning.db") + db = sqlite_utils.Database(log_path) + migrate(db) + db["responses"].insert( + { + "id": str(monotonic_ulid()).lower(), + "system": None, + "prompt": "hi", + "response": "answer", + "reasoning": "I thought hard about it.", + "model": "mock", + "datetime_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "conversation_id": "c1", + } + ) + runner = CliRunner() + result = runner.invoke(cli, ["logs", "-p", log_path], catch_exceptions=False) + assert result.exit_code == 0 + assert "## Reasoning\n\nI thought hard about it." in result.output + reasoning_pos = result.output.index("## Reasoning") + response_pos = result.output.index("## Response") + assert reasoning_pos < response_pos + + +def test_logs_markdown_omits_reasoning_heading_when_empty(log_path): + """When reasoning is empty/null, no heading appears (existing + fixture rows have no reasoning).""" + runner = CliRunner() + result = runner.invoke(cli, ["logs", "-p", str(log_path)], catch_exceptions=False) + assert result.exit_code == 0 + assert "## Reasoning" not in result.output diff --git a/tests/test_migrate.py b/tests/test_migrate.py index c526117eb..705021100 100644 --- a/tests/test_migrate.py +++ b/tests/test_migrate.py @@ -21,6 +21,7 @@ "output_tokens": int, "token_details": str, "schema_id": str, + "reasoning": str, } From beaec1e20c59a7ef9222adc0264aca8427ad6009 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 14:27:30 -0700 Subject: [PATCH 057/258] Lint fixes: mypy, ruff, black - Drop the placeholder messages() declaration on _BaseResponse so AsyncResponse.messages() (an async coroutine) no longer trips the mypy override check. text/json/tool_calls already follow this pattern. - Remove three unused imports flagged by ruff in test_parts.py. - Apply pending black reformats across the tree. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/embeddings.py | 4 +++- llm/embeddings_migrations.py | 12 ++++++++---- llm/models.py | 4 ---- tests/test_fragments_cli.py | 8 ++++++-- tests/test_parts.py | 8 ++------ tests/test_plugins.py | 6 ++++-- tests/test_templates.py | 6 ++++-- 7 files changed, 27 insertions(+), 21 deletions(-) diff --git a/llm/embeddings.py b/llm/embeddings.py index 90b983a11..5c9bf8ffa 100644 --- a/llm/embeddings.py +++ b/llm/embeddings.py @@ -202,7 +202,9 @@ def embed_multi_with_metadata( """ select id from embeddings where collection_id = ? and content_hash in ({}) - """.format(",".join("?" for _ in items_and_hashes)), + """.format( + ",".join("?" for _ in items_and_hashes) + ), [collection_id] + [item_and_hash[1] for item_and_hash in items_and_hashes], ) diff --git a/llm/embeddings_migrations.py b/llm/embeddings_migrations.py index 69545f3ea..600ad204d 100644 --- a/llm/embeddings_migrations.py +++ b/llm/embeddings_migrations.py @@ -63,16 +63,20 @@ def random_md5(): db.conn.create_function("temp_random_md5", 0, random_md5) with db.conn: - db.execute(""" + db.execute( + """ update embeddings set content_hash = temp_md5(content) where content is not null - """) - db.execute(""" + """ + ) + db.execute( + """ update embeddings set content_hash = temp_random_md5() where content is null - """) + """ + ) db["embeddings"].create_index(["content_hash"]) diff --git a/llm/models.py b/llm/models.py index 2ca64846a..7829a1855 100644 --- a/llm/models.py +++ b/llm/models.py @@ -888,10 +888,6 @@ def __init__( if self.prompt.tools and not self.model.supports_tools: raise ValueError(f"{self.model} does not support tools") - def messages(self) -> List[Any]: - "Overridden by Response / AsyncResponse — declared here for type checkers." - raise NotImplementedError - def _messages_now(self) -> List[Any]: """Assemble messages assuming the response is already drained. diff --git a/tests/test_fragments_cli.py b/tests/test_fragments_cli.py index 5975c9e70..8606205bd 100644 --- a/tests/test_fragments_cli.py +++ b/tests/test_fragments_cli.py @@ -106,7 +106,9 @@ def test_fragments_list(user_path): ) result = runner.invoke(cli, ["fragments", "list"]) assert result.exit_code == 0 - assert result.output.strip() == (textwrap.dedent(""" + assert result.output.strip() == ( + textwrap.dedent( + """ - hash: hash2 aliases: [] datetime_utc: '2022-10-01T00:00:00Z' @@ -123,7 +125,9 @@ def test_fragments_list(user_path): datetime_utc: '2024-10-01T00:00:00Z' source: file3.txt content: '3' - """).strip()) + """ + ).strip() + ) @mock.patch.dict(os.environ, {"OPENAI_API_KEY": "X"}) diff --git a/tests/test_parts.py b/tests/test_parts.py index 59dafb093..097f20bbc 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -1276,11 +1276,7 @@ def execute(self, prompt, stream, response, conversation): ] def test_reply_with_tool_results_and_prompt(self, mock_model): - from llm.parts import ( - Message, - ToolCallPart, - ToolResultPart, - ) + from llm.parts import ToolResultPart class ToolCallMock(type(mock_model)): supports_tools = True @@ -1318,7 +1314,7 @@ def execute(self, prompt, stream, response, conversation): def test_reply_auto_executes_tool_calls_when_none_passed(self, mock_model): # Zero-arg sugar: response.reply() with tool calls present # auto-executes them and threads results back into the chain. - from llm.parts import Message, ToolResultPart + from llm.parts import ToolResultPart executed = [] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 6777fd585..52203925f 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -174,7 +174,8 @@ def register_fragment_loaders(self, register): cli.cli, ["-m", "echo", "-f", "mixed:x"], catch_exceptions=False ) assert result3.exit_code == 0 - result3.output.strip == textwrap.dedent("""\ + result3.output.strip == textwrap.dedent( + """\ system: @@ -183,7 +184,8 @@ def register_fragment_loaders(self, register): attachments: - https://example.com/attachment.png - """).strip() + """ + ).strip() finally: plugins.pm.unregister(name="FragmentLoadersPlugin") diff --git a/tests/test_templates.py b/tests/test_templates.py index 38229619b..d63187d40 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -464,7 +464,8 @@ def register_tools(self, register): def test_tools_in_templates( source, expected_tool_success, expected_functions_success, httpx_mock, tmpdir ): - template_yaml = textwrap.dedent(""" + template_yaml = textwrap.dedent( + """ name: test tools: - llm_version @@ -472,7 +473,8 @@ def test_tools_in_templates( functions: | def demo(): return "Demo" - """) + """ + ) args = [] def before(): From 49dd7962646b8a1c797418d4a13a94dedf9eac95 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 14:39:56 -0700 Subject: [PATCH 058/258] Strip trailing whitespace from reasoning when rendering markdown logs Providers (e.g. Gemini) often emit thought text with trailing newlines. Concatenated and combined with click.echo's own newline, that produced several blank lines before the `## Response` heading. rstrip() at render time tightens the gap to a single blank line; the stored reasoning column keeps the provider's text verbatim. Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/cli.py | 2 +- tests/test_llm_logs.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index a53b54063..6d52a94c9 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -2223,7 +2223,7 @@ def _display_fragments(fragments, title): except ValueError: pass if row.get("reasoning"): - click.echo("\n## Reasoning\n\n{}".format(row["reasoning"])) + click.echo("\n## Reasoning\n\n{}".format(row["reasoning"].rstrip())) click.echo("\n## Response\n") if row["tool_calls"]: click.echo("### Tool calls\n") diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index 3a37e117a..bf55a162b 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -1059,7 +1059,7 @@ def test_logs_markdown_renders_reasoning_heading(user_path): "system": None, "prompt": "hi", "response": "answer", - "reasoning": "I thought hard about it.", + "reasoning": "I thought hard about it.\n\n\n", "model": "mock", "datetime_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), "conversation_id": "c1", @@ -1068,10 +1068,9 @@ def test_logs_markdown_renders_reasoning_heading(user_path): runner = CliRunner() result = runner.invoke(cli, ["logs", "-p", log_path], catch_exceptions=False) assert result.exit_code == 0 - assert "## Reasoning\n\nI thought hard about it." in result.output - reasoning_pos = result.output.index("## Reasoning") - response_pos = result.output.index("## Response") - assert reasoning_pos < response_pos + # rstrip() before rendering so trailing newlines from the + # provider output don't push `## Response` down the page. + assert "## Reasoning\n\nI thought hard about it.\n\n## Response" in result.output def test_logs_markdown_omits_reasoning_heading_when_empty(log_path): From 0fa7ccf58fbf8a7dcded7adff43e73ac3e5d7b67 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 14:42:39 -0700 Subject: [PATCH 059/258] Ran Black --- llm/cli.py | 32 ++++++++++---------------------- llm/embeddings.py | 4 +--- llm/embeddings_migrations.py | 12 ++++-------- tests/test_chat.py | 6 ++---- tests/test_fragments_cli.py | 8 ++------ tests/test_llm_logs.py | 6 ++---- tests/test_plugins.py | 6 ++---- tests/test_templates.py | 6 ++---- 8 files changed, 25 insertions(+), 55 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index 6d52a94c9..386b35bdb 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -1767,16 +1767,14 @@ def logs_list( if any_tools: # Any response that involved at least one tool result - where_bits.append( - """ + where_bits.append(""" exists ( select 1 from tool_results where tool_results.response_id = responses.id ) - """ - ) + """) if tools: tools_by_name = get_tools() # Filter responses by tools (must have ALL of the named tools, including plugin) @@ -1787,8 +1785,7 @@ def logs_list( except KeyError: raise click.ClickException(f"Unknown tool: {tool_name}") - tool_clauses.append( - f""" + tool_clauses.append(f""" exists ( select 1 from tool_results @@ -1797,8 +1794,7 @@ def logs_list( and tools.name = :tool{i} and tools.plugin = :plugin{i} ) - """ - ) + """) sql_params[f"tool{i}"] = tool_name sql_params[f"plugin{i}"] = plugin_name @@ -2530,9 +2526,7 @@ def schemas_list(path, database, queries, full, json_, nl): on responses.schema_id = schemas.id {} group by responses.schema_id order by recently_used - """.format( - where_sql - ) + """.format(where_sql) rows = db.query(sql, params) if json_ or nl: @@ -2871,13 +2865,11 @@ def fragments_list(queries, aliases, json_): param_count += 1 p = f"p{param_count}" params[p] = q - where_bits.append( - f""" + where_bits.append(f""" (fragments.hash = :{p} or fragment_aliases.alias = :{p} or fragments.source like '%' || :{p} || '%' or fragments.content like '%' || :{p} || '%') - """ - ) + """) where = "\n and\n ".join(where_bits) if where: where = " where " + where @@ -2899,9 +2891,7 @@ def fragments_list(queries, aliases, json_): group by fragments.id, fragments.hash, fragments.content, fragments.datetime_utc, fragments.source order by fragments.datetime_utc - """.format( - where=where - ) + """.format(where=where) results = list(db.query(sql, params)) for result in results: result["aliases"] = json.loads(result["aliases"]) @@ -3591,8 +3581,7 @@ def embed_db_collections(database, json_): db = sqlite_utils.Database(str(database)) if not db["collections"].exists(): raise click.ClickException("No collections table found in {}".format(database)) - rows = db.query( - """ + rows = db.query(""" select collections.name, collections.model, @@ -3602,8 +3591,7 @@ def embed_db_collections(database, json_): on collections.id = embeddings.collection_id group by collections.name, collections.model - """ - ) + """) if json_: click.echo(json.dumps(list(rows), indent=4)) else: diff --git a/llm/embeddings.py b/llm/embeddings.py index 5c9bf8ffa..90b983a11 100644 --- a/llm/embeddings.py +++ b/llm/embeddings.py @@ -202,9 +202,7 @@ def embed_multi_with_metadata( """ select id from embeddings where collection_id = ? and content_hash in ({}) - """.format( - ",".join("?" for _ in items_and_hashes) - ), + """.format(",".join("?" for _ in items_and_hashes)), [collection_id] + [item_and_hash[1] for item_and_hash in items_and_hashes], ) diff --git a/llm/embeddings_migrations.py b/llm/embeddings_migrations.py index 600ad204d..69545f3ea 100644 --- a/llm/embeddings_migrations.py +++ b/llm/embeddings_migrations.py @@ -63,20 +63,16 @@ def random_md5(): db.conn.create_function("temp_random_md5", 0, random_md5) with db.conn: - db.execute( - """ + db.execute(""" update embeddings set content_hash = temp_md5(content) where content is not null - """ - ) - db.execute( - """ + """) + db.execute(""" update embeddings set content_hash = temp_random_md5() where content is null - """ - ) + """) db["embeddings"].create_index(["content_hash"]) diff --git a/tests/test_chat.py b/tests/test_chat.py index e455bd2f0..5d6089cb0 100644 --- a/tests/test_chat.py +++ b/tests/test_chat.py @@ -314,13 +314,11 @@ def test_llm_chat_creates_log_database(tmpdir, monkeypatch, custom_database_path @pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows") def test_chat_tools(logs_db): runner = CliRunner() - functions = textwrap.dedent( - """ + functions = textwrap.dedent(""" def upper(text: str) -> str: "Convert text to upper case" return text.upper() - """ - ) + """) result = runner.invoke( llm.cli.cli, ["chat", "-m", "echo", "--functions", functions], diff --git a/tests/test_fragments_cli.py b/tests/test_fragments_cli.py index 8606205bd..5975c9e70 100644 --- a/tests/test_fragments_cli.py +++ b/tests/test_fragments_cli.py @@ -106,9 +106,7 @@ def test_fragments_list(user_path): ) result = runner.invoke(cli, ["fragments", "list"]) assert result.exit_code == 0 - assert result.output.strip() == ( - textwrap.dedent( - """ + assert result.output.strip() == (textwrap.dedent(""" - hash: hash2 aliases: [] datetime_utc: '2022-10-01T00:00:00Z' @@ -125,9 +123,7 @@ def test_fragments_list(user_path): datetime_utc: '2024-10-01T00:00:00Z' source: file3.txt content: '3' - """ - ).strip() - ) + """).strip()) @mock.patch.dict(os.environ, {"OPENAI_API_KEY": "X"}) diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index bf55a162b..f3c690b43 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -934,12 +934,10 @@ def test_expand_fragment_markdown(fragments_fixture): def test_logs_tools(logs_db): runner = CliRunner() - code = textwrap.dedent( - """ + code = textwrap.dedent(""" def demo(): return "one\\ntwo\\nthree" - """ - ) + """) result1 = runner.invoke( cli, [ diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 52203925f..6777fd585 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -174,8 +174,7 @@ def register_fragment_loaders(self, register): cli.cli, ["-m", "echo", "-f", "mixed:x"], catch_exceptions=False ) assert result3.exit_code == 0 - result3.output.strip == textwrap.dedent( - """\ + result3.output.strip == textwrap.dedent("""\ system: @@ -184,8 +183,7 @@ def register_fragment_loaders(self, register): attachments: - https://example.com/attachment.png - """ - ).strip() + """).strip() finally: plugins.pm.unregister(name="FragmentLoadersPlugin") diff --git a/tests/test_templates.py b/tests/test_templates.py index d63187d40..38229619b 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -464,8 +464,7 @@ def register_tools(self, register): def test_tools_in_templates( source, expected_tool_success, expected_functions_success, httpx_mock, tmpdir ): - template_yaml = textwrap.dedent( - """ + template_yaml = textwrap.dedent(""" name: test tools: - llm_version @@ -473,8 +472,7 @@ def test_tools_in_templates( functions: | def demo(): return "Demo" - """ - ) + """) args = [] def before(): From 29e3787be5e00a0fced6298af979323157904f34 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 16:32:52 -0700 Subject: [PATCH 060/258] Draft changelog for 0.32a0 --- docs/changelog.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 563ec4133..8ffe9d319 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,47 @@ # Changelog +(v0_32_a0)= +## 0.32a0 (2026-04-28) + +This alpha introduces a new structured representation for LLM conversations. Prompt inputs and response outputs can now be expressed as a list of `Message` objects, each containing typed `Part` objects (text, reasoning, tool calls, tool results, attachments). The previous string-based API continues to work unchanged. + +Plugin authors should read the substantially expanded {ref}`Advanced model plugins ` documentation, which now covers `StreamEvent`, consuming `prompt.messages`, and round-tripping opaque provider metadata such as Anthropic extended-thinking signatures and Gemini `thoughtSignature` values. + +### Structured messages and streaming events + +- New value types exported from the top-level `llm` package: `Message`, `Part`, `TextPart`, `ReasoningPart`, `ToolCallPart`, `ToolResultPart`, `AttachmentPart`, and `StreamEvent`. Plus constructor helpers `llm.user()`, `llm.assistant()`, `llm.system()`, and `llm.tool_message()` that accept strings, `Attachment` instances, or nested `Part` lists. +- New `messages=` keyword argument on `model.prompt()`, `conversation.prompt()`, `model.chain()`, `conversation.chain()`, and their async counterparts. When provided, `messages=` is the authoritative input to the model — `prompt.messages` returns it verbatim. The legacy `prompt=`, `system=`, `attachments=`, and `tool_results=` keywords still work and synthesize into the same `Message` list internally. +- New `response.stream_events()` and `response.astream_events()` methods yielding typed `StreamEvent` objects (`type` is one of `"text"`, `"reasoning"`, `"tool_call_name"`, `"tool_call_args"`, `"tool_result"`, plus a `redacted=True` marker for opaque reasoning). Each event carries a `part_index` that groups events into the same logical Part. Iterating against `response` directly continues to yield only text strings. +- New `response.messages()` method (async: `await response.messages()`) returning the assembled `list[Message]` produced by the model. Calling it forces execution if the response has not yet been drained, matching the `.text()` / `.json()` / `.tool_calls()` pattern. +- New `response.reply(prompt=None, **kwargs)` method that continues the conversation from any `Response`, regardless of origin. When the previous response made tool calls and `tool_results=` was not passed, `reply()` automatically executes the pending tool calls and threads the results into the next turn. On async responses `reply()` is awaitable. +- New `response.to_dict()` and `Response.from_dict(data, *, model=None)` for JSON-safe serialization of a full conversation turn — model id, input chain, assembled output (including reasoning parts and provider metadata), options, and audit fields. Reasoning signatures and `thoughtSignature` values round-trip via `provider_metadata`, so multi-turn extended thinking works across process boundaries. +- New `llm/serialization.py` module exposing `MessageDict`, `PartDict`, `ResponseDict`, `PromptDict`, `UsageDict`, `AttachmentDict`, and the per-Part TypedDicts. Every `to_dict()` / `from_dict()` method is annotated with the matching TypedDict, so consumers get autocomplete, mypy coverage, and pydantic `TypeAdapter` validation for free. +- `Response.prompt.messages` is now the canonical structured input across the entire conversation chain. `Conversation.prompt` and `AsyncConversation.prompt` pre-compute the full chain (prior input + prior output + new turn) before constructing the next `Prompt`, so `response.prompt.messages` is always exactly what the model was sent. + +### CLI + +- `llm prompt` and `llm chat` now display visible reasoning text to stderr in a dim style while the response streams, with a newline inserted at the reasoning-to-text transition so the assistant's final answer starts on a fresh line. +- New `-R/--no-reasoning` flag for `llm prompt` and `llm chat` to suppress the reasoning stream. +- `llm logs --md` now renders any visible reasoning emitted during a response under a `## Reasoning` heading above the response. +- New `reasoning` column on the `responses` table populated from the visible-reasoning text. Added by migration `m022`. + +### OpenAI plugin + +- Rewritten to consume `prompt.messages` and dispatch per-`Part` into OpenAI's wire format. The legacy `prompt=` / `system=` / `attachments=` / `tool_results=` entry points continue to work because `Prompt.messages` synthesizes those into the same `Message` list the new code reads. +- `Chat.execute()` and `AsyncChat.execute()` yield `StreamEvent` objects (text, tool-call name, tool-call argument deltas) for both streaming and non-streaming paths. Empty-string content deltas (OpenAI's first `role=assistant` chunk) are now skipped as noise. +- Reasoning token counts from OpenAI reasoning models (e.g. GPT-5 family) are captured before `set_usage` mutates the usage dict and surfaced as a redacted `ReasoningPart` on the assembled message, so callers can render "the model used N reasoning tokens here" without any visible reasoning text being available. + +### Documentation + +- Major expansion of {ref}`Advanced model plugins ` covering the `StreamEvent` contract, `part_index` allocation rules, server-executed tools, opaque `provider_metadata`, the new `prompt.messages` invariant for `build_messages`, role mapping across OpenAI / Anthropic / Gemini conventions, and how to fold reasoning signatures and `thoughtSignature` values back into outgoing requests. +- New {ref}`Structured messages and streaming events ` section in the Python API docs walking through `messages=[...]`, `stream_events()`, `response.messages()`, `response.reply()`, and the `to_dict()` / `from_dict()` round-trip pattern. + +### Bug fixes + +- Parallel tool calls emitted without a `tool_call_id` (e.g. Gemini's parallel function calls) are no longer collapsed into a single `ToolCallPart` with concatenated names and arguments. Each fresh `tool_call_name` event now allocates a new `part_index`, producing one `ToolCallPart` per call. +- Tool-result turns inside a `chain()` loop now carry `system=` and `system_fragments=` forward from the initial prompt. Adapters that read `prompt.system` directly (e.g. OpenAI's Chat completions, which sends system as its own message) previously saw an empty system on every turn after the first. +- Fixed a regression where `llm -c` (and `load_conversation().prompt(...)`) dropped the prior assistant turn from the chain when the response was rehydrated from SQLite. `response.messages` now falls back to synthesizing from the persisted text and tool-call columns when no `StreamEvent`s are available. + (v0_31)= ## 0.31 (2026-04-24) From 7c471496f4dca798196ec505c4f643f032ae0ddc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 16:44:21 -0700 Subject: [PATCH 061/258] Do not have xPart classes as llm. imports --- llm/__init__.py | 14 -------------- llm/default_plugins/openai_models.py | 2 +- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/llm/__init__.py b/llm/__init__.py index bb84c3911..5dd52a273 100644 --- a/llm/__init__.py +++ b/llm/__init__.py @@ -27,14 +27,7 @@ Usage, ) from .parts import ( - AttachmentPart, Message, - Part, - ReasoningPart, - StreamEvent, - TextPart, - ToolCallPart, - ToolResultPart, assistant, system, tool_message, @@ -59,7 +52,6 @@ "AsyncResponse", "assistant", "Attachment", - "AttachmentPart", "CancelToolCall", "Collection", "Conversation", @@ -74,23 +66,17 @@ "ModelError", "NeedsKeyException", "Options", - "Part", "Prompt", - "ReasoningPart", "Response", "schema_dsl", - "StreamEvent", "system", "Template", - "TextPart", "Tool", "Toolbox", "ToolCall", - "ToolCallPart", "tool_message", "ToolOutput", "ToolResult", - "ToolResultPart", "Usage", "user", "user_dir", diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index eddd489ae..8e45c1145 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -7,10 +7,10 @@ KeyModel, Prompt, Response, - StreamEvent, hookimpl, ) import llm +from llm.parts import StreamEvent from llm.utils import ( dicts_to_table_string, remove_dict_none_values, From 3497c22e8c122f04fbae1fb156bdec486203bde1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 16:48:48 -0700 Subject: [PATCH 062/258] Black --- tests/test_async_parity.py | 2 +- tests/test_cli_streaming.py | 30 ++- tests/test_llm_logs.py | 6 +- tests/test_openai_messages.py | 36 +-- tests/test_parts.py | 412 ++++++++++++++++++---------------- tests/test_serialization.py | 50 +++-- 6 files changed, 288 insertions(+), 248 deletions(-) diff --git a/tests/test_async_parity.py b/tests/test_async_parity.py index 68ea1e3ac..0ccaa8394 100644 --- a/tests/test_async_parity.py +++ b/tests/test_async_parity.py @@ -111,7 +111,7 @@ async def test_async_from_row_response_messages_synthesized(tmp_path): msgs = await rehydrated.messages() assert len(msgs) == 1 assert msgs[0].role == "assistant" - assert isinstance(msgs[0].parts[0], llm.TextPart) + assert isinstance(msgs[0].parts[0], llm.parts.TextPart) # ---- AsyncConversation follow-up via load_conversation ------------- diff --git a/tests/test_cli_streaming.py b/tests/test_cli_streaming.py index c6ad624e7..c8efee2fe 100644 --- a/tests/test_cli_streaming.py +++ b/tests/test_cli_streaming.py @@ -26,8 +26,10 @@ def test_text_goes_to_stdout_not_stderr(mock_model): def test_reasoning_goes_to_stderr_not_stdout(mock_model): mock_model.enqueue( [ - llm.StreamEvent(type="reasoning", chunk="thinking hard", part_index=0), - llm.StreamEvent(type="text", chunk="answer", part_index=1), + llm.parts.StreamEvent( + type="reasoning", chunk="thinking hard", part_index=0 + ), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), ] ) runner = CliRunner(mix_stderr=False) @@ -46,8 +48,8 @@ def test_reasoning_rendered_in_dim_style(mock_model): """The click.style(..., dim=True) wrapper emits the ANSI dim code.""" mock_model.enqueue( [ - llm.StreamEvent(type="reasoning", chunk="t", part_index=0), - llm.StreamEvent(type="text", chunk="x", part_index=1), + llm.parts.StreamEvent(type="reasoning", chunk="t", part_index=0), + llm.parts.StreamEvent(type="text", chunk="x", part_index=1), ] ) runner = CliRunner(mix_stderr=False) @@ -66,8 +68,10 @@ def test_reasoning_rendered_in_dim_style(mock_model): def test_no_reasoning_flag_suppresses_reasoning(mock_model): mock_model.enqueue( [ - llm.StreamEvent(type="reasoning", chunk="hidden thinking", part_index=0), - llm.StreamEvent(type="text", chunk="answer", part_index=1), + llm.parts.StreamEvent( + type="reasoning", chunk="hidden thinking", part_index=0 + ), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), ] ) runner = CliRunner(mix_stderr=False) @@ -85,8 +89,8 @@ def test_no_reasoning_flag_suppresses_reasoning(mock_model): def test_no_reasoning_short_flag_R(mock_model): mock_model.enqueue( [ - llm.StreamEvent(type="reasoning", chunk="hidden", part_index=0), - llm.StreamEvent(type="text", chunk="x", part_index=1), + llm.parts.StreamEvent(type="reasoning", chunk="hidden", part_index=0), + llm.parts.StreamEvent(type="text", chunk="x", part_index=1), ] ) runner = CliRunner(mix_stderr=False) @@ -104,8 +108,8 @@ def test_newline_between_reasoning_and_text(mock_model): text on stdout starts on a fresh visual line.""" mock_model.enqueue( [ - llm.StreamEvent(type="reasoning", chunk="think", part_index=0), - llm.StreamEvent(type="text", chunk="answer", part_index=1), + llm.parts.StreamEvent(type="reasoning", chunk="think", part_index=0), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), ] ) runner = CliRunner(mix_stderr=False) @@ -122,8 +126,10 @@ def test_newline_between_reasoning_and_text(mock_model): def test_async_path_reasoning_to_stderr(async_mock_model): async_mock_model.enqueue( [ - llm.StreamEvent(type="reasoning", chunk="async thinking", part_index=0), - llm.StreamEvent(type="text", chunk="async answer", part_index=1), + llm.parts.StreamEvent( + type="reasoning", chunk="async thinking", part_index=0 + ), + llm.parts.StreamEvent(type="text", chunk="async answer", part_index=1), ] ) runner = CliRunner(mix_stderr=False) diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index f3c690b43..f2f6167c6 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -1021,9 +1021,9 @@ def test_log_to_db_persists_visible_reasoning(logs_db, mock_model): mock_model.enqueue( [ - llm.StreamEvent(type="reasoning", chunk="thinking "), - llm.StreamEvent(type="reasoning", chunk="hard"), - llm.StreamEvent(type="text", chunk="hello"), + llm.parts.StreamEvent(type="reasoning", chunk="thinking "), + llm.parts.StreamEvent(type="reasoning", chunk="hard"), + llm.parts.StreamEvent(type="text", chunk="hello"), ] ) response = mock_model.prompt("hi") diff --git a/tests/test_openai_messages.py b/tests/test_openai_messages.py index e9341eab5..9d2ae911d 100644 --- a/tests/test_openai_messages.py +++ b/tests/test_openai_messages.py @@ -136,7 +136,7 @@ def test_user_with_attachment(self, chat_model): ] def test_assistant_with_tool_call(self, chat_model): - tool_call = llm.ToolCallPart( + tool_call = llm.parts.ToolCallPart( name="search", arguments={"q": "weather"}, tool_call_id="c1", @@ -171,7 +171,7 @@ def test_assistant_with_tool_call(self, chat_model): def test_assistant_tool_call_only_no_text(self, chat_model): """When an assistant message has tool_calls but no text, OpenAI expects content=null.""" - tool_call = llm.ToolCallPart( + tool_call = llm.parts.ToolCallPart( name="search", arguments={"q": "x"}, tool_call_id="c1" ) prompt = Prompt( @@ -196,7 +196,7 @@ def test_assistant_tool_call_only_no_text(self, chat_model): } def test_tool_role_message_with_tool_result(self, chat_model): - tr = llm.ToolResultPart(name="search", output="sunny", tool_call_id="c1") + tr = llm.parts.ToolResultPart(name="search", output="sunny", tool_call_id="c1") prompt = Prompt( None, model=chat_model, @@ -213,8 +213,8 @@ def test_tool_role_message_with_tool_result(self, chat_model): def test_multiple_tool_results_emit_multiple_messages(self, chat_model): """Parallel tool results: one OpenAI 'tool' message per result.""" - a = llm.ToolResultPart(name="t", output="A", tool_call_id="c1") - b = llm.ToolResultPart(name="t", output="B", tool_call_id="c2") + a = llm.parts.ToolResultPart(name="t", output="A", tool_call_id="c1") + b = llm.parts.ToolResultPart(name="t", output="B", tool_call_id="c2") prompt = Prompt( None, model=chat_model, @@ -396,7 +396,7 @@ def test_text_stream_yields_text_events(self, httpx_mock): events = list(response.stream_events()) # At least one StreamEvent, all text, all at part_index=0. assert events, "expected stream events" - assert all(isinstance(e, llm.StreamEvent) for e in events) + assert all(isinstance(e, llm.parts.StreamEvent) for e in events) assert all(e.type == "text" for e in events) assert all(e.part_index == 0 for e in events) # Text chunks concatenate to the expected full text. @@ -426,7 +426,7 @@ def test_text_stream_messages_assembled(self, httpx_mock): response = model.prompt("hi", key=API_KEY) response.text() assert response.messages() == [ - llm.Message(role="assistant", parts=[llm.TextPart(text="Hello")]) + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="Hello")]) ] def test_tool_call_stream_yields_name_and_args_events(self, httpx_mock): @@ -496,10 +496,10 @@ def get_weather(c: int) -> str: response.text() # After streaming, messages has both a TextPart and a ToolCallPart. parts = response.messages()[0].parts - assert any(isinstance(p, llm.TextPart) for p in parts) - assert any(isinstance(p, llm.ToolCallPart) for p in parts) - text_part = next(p for p in parts if isinstance(p, llm.TextPart)) - tc_part = next(p for p in parts if isinstance(p, llm.ToolCallPart)) + assert any(isinstance(p, llm.parts.TextPart) for p in parts) + assert any(isinstance(p, llm.parts.ToolCallPart) for p in parts) + text_part = next(p for p in parts if isinstance(p, llm.parts.TextPart)) + tc_part = next(p for p in parts if isinstance(p, llm.parts.ToolCallPart)) assert text_part.text == "Looking up" assert tc_part.name == "get_weather" assert tc_part.arguments == {"c": 1} @@ -519,7 +519,7 @@ async def test_text_stream_yields_text_events(self, httpx_mock): events = [] async for event in response.astream_events(): events.append(event) - assert all(isinstance(e, llm.StreamEvent) for e in events) + assert all(isinstance(e, llm.parts.StreamEvent) for e in events) assert [e.type for e in events] == ["text"] * len(events) assert "".join(e.chunk for e in events) == "Hello" @@ -559,8 +559,8 @@ def test_redacted_reasoning_part_emitted_when_count_present(self, httpx_mock): llm.Message( role="assistant", parts=[ - llm.ReasoningPart(text="", redacted=True), - llm.TextPart(text="Hello"), + llm.parts.ReasoningPart(text="", redacted=True), + llm.parts.TextPart(text="Hello"), ], ) ] @@ -577,7 +577,7 @@ def test_no_reasoning_part_when_zero_or_absent(self, httpx_mock): response.text() parts = response.messages()[0].parts assert not any( - isinstance(p, llm.ReasoningPart) for p in parts + isinstance(p, llm.parts.ReasoningPart) for p in parts ), "should not add a redacted reasoning part when count=0" @@ -605,7 +605,9 @@ def test_non_streaming_text_yields_single_event(self, httpx_mock): model = llm.get_model("gpt-4o-mini") response = model.prompt("hi", key=API_KEY, stream=False) events = list(response.stream_events()) - assert events == [llm.StreamEvent(type="text", chunk="Hello", part_index=0)] + assert events == [ + llm.parts.StreamEvent(type="text", chunk="Hello", part_index=0) + ] assert response.messages() == [ - llm.Message(role="assistant", parts=[llm.TextPart(text="Hello")]) + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="Hello")]) ] diff --git a/tests/test_parts.py b/tests/test_parts.py index 097f20bbc..e23f92d04 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -5,57 +5,59 @@ class TestTextPart: def test_roundtrip(self): - part = llm.TextPart(text="Hello world") - restored = llm.Part.from_dict(part.to_dict()) + part = llm.parts.TextPart(text="Hello world") + restored = llm.parts.Part.from_dict(part.to_dict()) assert restored == part - assert isinstance(restored, llm.TextPart) + assert isinstance(restored, llm.parts.TextPart) assert restored.text == "Hello world" def test_to_dict_shape(self): - assert llm.TextPart(text="hi").to_dict() == {"type": "text", "text": "hi"} + assert llm.parts.TextPart(text="hi").to_dict() == {"type": "text", "text": "hi"} def test_with_provider_metadata(self): - part = llm.TextPart(text="hi", provider_metadata={"openai": {"flag": True}}) - restored = llm.Part.from_dict(part.to_dict()) + part = llm.parts.TextPart( + text="hi", provider_metadata={"openai": {"flag": True}} + ) + restored = llm.parts.Part.from_dict(part.to_dict()) assert restored == part class TestReasoningPart: def test_roundtrip_with_text(self): - part = llm.ReasoningPart(text="Let me think...") - restored = llm.Part.from_dict(part.to_dict()) + part = llm.parts.ReasoningPart(text="Let me think...") + restored = llm.parts.Part.from_dict(part.to_dict()) assert restored == part assert restored.text == "Let me think..." assert restored.redacted is False def test_roundtrip_redacted(self): - part = llm.ReasoningPart(text="", redacted=True) + part = llm.parts.ReasoningPart(text="", redacted=True) d = part.to_dict() assert d["redacted"] is True assert "token_count" not in d - restored = llm.Part.from_dict(d) + restored = llm.parts.Part.from_dict(d) assert restored == part def test_no_token_count_field(self): # token_count was removed: opaque token totals live on # response.token_details, not on the Part. with pytest.raises(TypeError): - llm.ReasoningPart(text="", redacted=True, token_count=150) + llm.parts.ReasoningPart(text="", redacted=True, token_count=150) class TestToolCallPart: def test_roundtrip(self): - part = llm.ToolCallPart( + part = llm.parts.ToolCallPart( name="search", arguments={"query": "weather"}, tool_call_id="call_123", ) - restored = llm.Part.from_dict(part.to_dict()) + restored = llm.parts.Part.from_dict(part.to_dict()) assert restored == part assert restored.server_executed is False def test_server_executed_flag_roundtrips(self): - part = llm.ToolCallPart( + part = llm.parts.ToolCallPart( name="web_search", arguments={"q": "x"}, tool_call_id="c1", @@ -63,45 +65,47 @@ def test_server_executed_flag_roundtrips(self): ) d = part.to_dict() assert d["server_executed"] is True - restored = llm.Part.from_dict(d) + restored = llm.parts.Part.from_dict(d) assert restored.server_executed is True class TestToolResultPart: def test_roundtrip(self): - part = llm.ToolResultPart(name="search", output="72F sunny", tool_call_id="c1") - restored = llm.Part.from_dict(part.to_dict()) + part = llm.parts.ToolResultPart( + name="search", output="72F sunny", tool_call_id="c1" + ) + restored = llm.parts.Part.from_dict(part.to_dict()) assert restored == part assert restored.exception is None assert restored.attachments == [] def test_with_exception(self): - part = llm.ToolResultPart( + part = llm.parts.ToolResultPart( name="t", output="", tool_call_id="c1", exception="boom" ) - restored = llm.Part.from_dict(part.to_dict()) + restored = llm.parts.Part.from_dict(part.to_dict()) assert restored.exception == "boom" class TestAttachmentPart: def test_roundtrip_with_url(self): att = llm.Attachment(url="http://example.com/cat.jpg") - part = llm.AttachmentPart(attachment=att) - restored = llm.Part.from_dict(part.to_dict()) - assert isinstance(restored, llm.AttachmentPart) + part = llm.parts.AttachmentPart(attachment=att) + restored = llm.parts.Part.from_dict(part.to_dict()) + assert isinstance(restored, llm.parts.AttachmentPart) assert restored.attachment.url == "http://example.com/cat.jpg" def test_roundtrip_with_path(self): att = llm.Attachment(type="image/jpeg", path="/tmp/x.jpg") - part = llm.AttachmentPart(attachment=att) - restored = llm.Part.from_dict(part.to_dict()) + part = llm.parts.AttachmentPart(attachment=att) + restored = llm.parts.Part.from_dict(part.to_dict()) assert restored.attachment.path == "/tmp/x.jpg" assert restored.attachment.type == "image/jpeg" def test_roundtrip_with_bytes_uses_base64(self): raw = b"\x89PNG fake bytes" att = llm.Attachment(type="image/png", content=raw) - part = llm.AttachmentPart(attachment=att) + part = llm.parts.AttachmentPart(attachment=att) d = part.to_dict() # Content must be a base64-encoded string in the dict form assert isinstance(d["attachment"]["content"], str) @@ -109,49 +113,49 @@ def test_roundtrip_with_bytes_uses_base64(self): assert base64.b64decode(d["attachment"]["content"]) == raw # And round-trip back to the original bytes - restored = llm.Part.from_dict(d) + restored = llm.parts.Part.from_dict(d) assert restored.attachment.content == raw def test_json_serializable(self): att = llm.Attachment(type="image/png", content=b"\x00\x01\x02") - part = llm.AttachmentPart(attachment=att) + part = llm.parts.AttachmentPart(attachment=att) # Must survive json dumps/loads - restored = llm.Part.from_dict(json.loads(json.dumps(part.to_dict()))) + restored = llm.parts.Part.from_dict(json.loads(json.dumps(part.to_dict()))) assert restored.attachment.content == b"\x00\x01\x02" class TestUnknownPart: def test_from_dict_unknown_type_raises(self): with pytest.raises(ValueError): - llm.Part.from_dict({"type": "nonsense"}) + llm.parts.Part.from_dict({"type": "nonsense"}) class TestRoleNotOnPart: def test_text_part_has_no_role_attribute(self): # Role lives on Message. Parts are content-only. - part = llm.TextPart(text="hi") + part = llm.parts.TextPart(text="hi") assert not hasattr(part, "role") def test_reasoning_part_has_no_role_attribute(self): - assert not hasattr(llm.ReasoningPart(text=""), "role") + assert not hasattr(llm.parts.ReasoningPart(text=""), "role") def test_tool_call_part_has_no_role_attribute(self): assert not hasattr( - llm.ToolCallPart(name="t", arguments={}, tool_call_id="c1"), + llm.parts.ToolCallPart(name="t", arguments={}, tool_call_id="c1"), "role", ) class TestMessage: def test_roundtrip_simple_user_message(self): - m = llm.Message(role="user", parts=[llm.TextPart(text="hi")]) + m = llm.Message(role="user", parts=[llm.parts.TextPart(text="hi")]) restored = llm.Message.from_dict(m.to_dict()) assert restored == m def test_roundtrip_with_provider_metadata(self): m = llm.Message( role="assistant", - parts=[llm.TextPart(text="hi")], + parts=[llm.parts.TextPart(text="hi")], provider_metadata={"anthropic": {"signature": "abc"}}, ) restored = llm.Message.from_dict(m.to_dict()) @@ -161,9 +165,9 @@ def test_roundtrip_mixed_parts(self): m = llm.Message( role="assistant", parts=[ - llm.ReasoningPart(text="Thinking"), - llm.TextPart(text="Result"), - llm.ToolCallPart( + llm.parts.ReasoningPart(text="Thinking"), + llm.parts.TextPart(text="Result"), + llm.parts.ToolCallPart( name="search", arguments={"q": "x"}, tool_call_id="c1", @@ -174,15 +178,15 @@ def test_roundtrip_mixed_parts(self): assert restored == m def test_empty_provider_metadata_omitted(self): - m = llm.Message(role="user", parts=[llm.TextPart(text="x")]) + m = llm.Message(role="user", parts=[llm.parts.TextPart(text="x")]) d = m.to_dict() assert "provider_metadata" not in d def test_none_and_empty_provider_metadata_equivalent(self): - m_none = llm.Message(role="user", parts=[llm.TextPart(text="x")]) + m_none = llm.Message(role="user", parts=[llm.parts.TextPart(text="x")]) m_empty = llm.Message( role="user", - parts=[llm.TextPart(text="x")], + parts=[llm.parts.TextPart(text="x")], provider_metadata={}, ) # Both serialize the same (empty metadata is omitted) @@ -193,20 +197,20 @@ class TestHelpers: def test_user_with_string(self): m = llm.user("hi") assert m.role == "user" - assert m.parts == [llm.TextPart(text="hi")] + assert m.parts == [llm.parts.TextPart(text="hi")] def test_assistant_with_string(self): m = llm.assistant("there") assert m.role == "assistant" - assert m.parts == [llm.TextPart(text="there")] + assert m.parts == [llm.parts.TextPart(text="there")] def test_system_with_string(self): m = llm.system("be brief") assert m.role == "system" - assert m.parts == [llm.TextPart(text="be brief")] + assert m.parts == [llm.parts.TextPart(text="be brief")] def test_tool_message_with_part(self): - tr = llm.ToolResultPart(name="t", output="r", tool_call_id="c1") + tr = llm.parts.ToolResultPart(name="t", output="r", tool_call_id="c1") m = llm.tool_message(tr) assert m.role == "tool" assert m.parts == [tr] @@ -215,12 +219,12 @@ def test_helper_accepts_attachment(self): att = llm.Attachment(url="http://example.com/x.jpg") m = llm.user("describe this", att) assert m.parts == [ - llm.TextPart(text="describe this"), - llm.AttachmentPart(attachment=att), + llm.parts.TextPart(text="describe this"), + llm.parts.AttachmentPart(attachment=att), ] def test_helper_accepts_existing_part(self): - tp = llm.TextPart(text="pre-built") + tp = llm.parts.TextPart(text="pre-built") m = llm.user(tp) assert m.parts == [tp] @@ -228,9 +232,9 @@ def test_helper_flattens_one_level(self): # Nested list gets flattened one level. m = llm.user(["one", "two"], "three") assert m.parts == [ - llm.TextPart(text="one"), - llm.TextPart(text="two"), - llm.TextPart(text="three"), + llm.parts.TextPart(text="one"), + llm.parts.TextPart(text="two"), + llm.parts.TextPart(text="three"), ] def test_helper_rejects_unknown_types(self): @@ -244,7 +248,7 @@ def test_helper_with_provider_metadata(self): class TestStreamEvent: def test_dataclass_defaults(self): - ev = llm.StreamEvent(type="text", chunk="hi", part_index=0) + ev = llm.parts.StreamEvent(type="text", chunk="hi", part_index=0) assert ev.type == "text" assert ev.chunk == "hi" assert ev.part_index == 0 @@ -255,7 +259,7 @@ def test_dataclass_defaults(self): assert ev.message_index == 0 def test_all_fields_accepted(self): - ev = llm.StreamEvent( + ev = llm.parts.StreamEvent( type="tool_call_args", chunk='{"q":', part_index=2, @@ -305,7 +309,7 @@ def test_stream_events_yields_text_events(self, mock_model): mock_model.enqueue(["hel", "lo"]) response = mock_model.prompt("hi") events = list(response.stream_events()) - assert all(isinstance(e, llm.StreamEvent) for e in events) + assert all(isinstance(e, llm.parts.StreamEvent) for e in events) assert [e.type for e in events] == ["text", "text"] assert [e.chunk for e in events] == ["hel", "lo"] assert all(e.part_index == 0 for e in events) @@ -316,7 +320,7 @@ def test_response_messages_is_single_assistant_text(self, mock_model): response.text() messages = response.messages() assert messages == [ - llm.Message(role="assistant", parts=[llm.TextPart(text="hello")]) + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="hello")]) ] def test_empty_response_has_empty_messages(self, mock_model): @@ -332,9 +336,9 @@ class TestStreamEventsFromStreamEventPlugin: def test_iter_yields_only_text_chunks(self, mock_model): events = [ - llm.StreamEvent(type="reasoning", chunk="think ", part_index=0), - llm.StreamEvent(type="text", chunk="hel", part_index=1), - llm.StreamEvent(type="text", chunk="lo", part_index=1), + llm.parts.StreamEvent(type="reasoning", chunk="think ", part_index=0), + llm.parts.StreamEvent(type="text", chunk="hel", part_index=1), + llm.parts.StreamEvent(type="text", chunk="lo", part_index=1), ] mock_model.enqueue(events) response = mock_model.prompt("hi") @@ -343,8 +347,8 @@ def test_iter_yields_only_text_chunks(self, mock_model): def test_stream_events_yields_all_events(self, mock_model): events = [ - llm.StreamEvent(type="reasoning", chunk="t", part_index=0), - llm.StreamEvent(type="text", chunk="x", part_index=1), + llm.parts.StreamEvent(type="reasoning", chunk="t", part_index=0), + llm.parts.StreamEvent(type="text", chunk="x", part_index=1), ] mock_model.enqueue(events) response = mock_model.prompt("hi") @@ -353,8 +357,8 @@ def test_stream_events_yields_all_events(self, mock_model): def test_messages_assembles_reasoning_then_text(self, mock_model): events = [ - llm.StreamEvent(type="reasoning", chunk="thinking", part_index=0), - llm.StreamEvent(type="text", chunk="hello", part_index=1), + llm.parts.StreamEvent(type="reasoning", chunk="thinking", part_index=0), + llm.parts.StreamEvent(type="text", chunk="hello", part_index=1), ] mock_model.enqueue(events) response = mock_model.prompt("hi") @@ -363,28 +367,28 @@ def test_messages_assembles_reasoning_then_text(self, mock_model): llm.Message( role="assistant", parts=[ - llm.ReasoningPart(text="thinking"), - llm.TextPart(text="hello"), + llm.parts.ReasoningPart(text="thinking"), + llm.parts.TextPart(text="hello"), ], ) ] def test_tool_call_name_and_args_merge(self, mock_model): events = [ - llm.StreamEvent(type="text", chunk="calling", part_index=0), - llm.StreamEvent( + llm.parts.StreamEvent(type="text", chunk="calling", part_index=0), + llm.parts.StreamEvent( type="tool_call_name", chunk="search", part_index=1, tool_call_id="c1", ), - llm.StreamEvent( + llm.parts.StreamEvent( type="tool_call_args", chunk='{"q":', part_index=1, tool_call_id="c1", ), - llm.StreamEvent( + llm.parts.StreamEvent( type="tool_call_args", chunk='"weather"}', part_index=1, @@ -398,8 +402,8 @@ def test_tool_call_name_and_args_merge(self, mock_model): assert len(msgs) == 1 parts = msgs[0].parts assert parts == [ - llm.TextPart(text="calling"), - llm.ToolCallPart( + llm.parts.TextPart(text="calling"), + llm.parts.ToolCallPart( name="search", arguments={"q": "weather"}, tool_call_id="c1", @@ -408,13 +412,13 @@ def test_tool_call_name_and_args_merge(self, mock_model): def test_tool_call_args_unparseable_json_falls_back(self, mock_model): events = [ - llm.StreamEvent( + llm.parts.StreamEvent( type="tool_call_name", chunk="t", part_index=0, tool_call_id="c1", ), - llm.StreamEvent( + llm.parts.StreamEvent( type="tool_call_args", chunk="not json", part_index=0, @@ -430,8 +434,8 @@ def test_tool_call_args_unparseable_json_falls_back(self, mock_model): def test_family_mismatch_at_same_part_index_raises(self, mock_model): events = [ - llm.StreamEvent(type="text", chunk="x", part_index=0), - llm.StreamEvent( + llm.parts.StreamEvent(type="text", chunk="x", part_index=0), + llm.parts.StreamEvent( type="tool_call_name", chunk="t", part_index=0, @@ -446,13 +450,13 @@ def test_family_mismatch_at_same_part_index_raises(self, mock_model): def test_provider_metadata_merges_last_wins(self, mock_model): events = [ - llm.StreamEvent( + llm.parts.StreamEvent( type="reasoning", chunk="think", part_index=0, provider_metadata={"anthropic": {"signature": "one"}}, ), - llm.StreamEvent( + llm.parts.StreamEvent( type="reasoning", chunk="", part_index=0, @@ -470,16 +474,16 @@ def test_redacted_reasoning_event_emits_marker_part(self, mock_model): # ReasoningPart(text="", redacted=True) marker — opaque token # totals live on response.token_details, not on the Part. events = [ - llm.StreamEvent(type="reasoning", chunk="", redacted=True), - llm.StreamEvent(type="text", chunk="hi"), + llm.parts.StreamEvent(type="reasoning", chunk="", redacted=True), + llm.parts.StreamEvent(type="text", chunk="hi"), ] mock_model.enqueue(events) response = mock_model.prompt("x") response.text() parts = response.messages()[0].parts assert parts == [ - llm.ReasoningPart(text="", redacted=True), - llm.TextPart(text="hi"), + llm.parts.ReasoningPart(text="", redacted=True), + llm.parts.TextPart(text="hi"), ] def test_redacted_reasoning_hoisted_to_start_when_emitted_late(self, mock_model): @@ -488,20 +492,20 @@ def test_redacted_reasoning_hoisted_to_start_when_emitted_late(self, mock_model) # The framework hoists redacted reasoning Parts to the start of # the assembled message so UIs can render them before content. events = [ - llm.StreamEvent(type="text", chunk="hello"), - llm.StreamEvent(type="reasoning", chunk="", redacted=True), + llm.parts.StreamEvent(type="text", chunk="hello"), + llm.parts.StreamEvent(type="reasoning", chunk="", redacted=True), ] mock_model.enqueue(events) response = mock_model.prompt("x") response.text() parts = response.messages()[0].parts assert parts == [ - llm.ReasoningPart(text="", redacted=True), - llm.TextPart(text="hello"), + llm.parts.ReasoningPart(text="", redacted=True), + llm.parts.TextPart(text="hello"), ] def test_redacted_reasoning_event_default_redacted_is_false(self): - ev = llm.StreamEvent(type="reasoning", chunk="thinking") + ev = llm.parts.StreamEvent(type="reasoning", chunk="thinking") assert ev.redacted is False @@ -511,69 +515,71 @@ class TestPartIndexAutoAllocation: and tool_call_id for tool calls.""" def test_streamevent_part_index_defaults_to_none(self): - ev = llm.StreamEvent(type="text", chunk="hi") + ev = llm.parts.StreamEvent(type="text", chunk="hi") assert ev.part_index is None def test_consecutive_text_concatenates_into_one_part(self, mock_model): events = [ - llm.StreamEvent(type="text", chunk="hello "), - llm.StreamEvent(type="text", chunk="world"), + llm.parts.StreamEvent(type="text", chunk="hello "), + llm.parts.StreamEvent(type="text", chunk="world"), ] mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() - assert response.messages()[0].parts == [llm.TextPart(text="hello world")] + assert response.messages()[0].parts == [llm.parts.TextPart(text="hello world")] def test_text_then_reasoning_splits_into_two_parts(self, mock_model): events = [ - llm.StreamEvent(type="text", chunk="hello"), - llm.StreamEvent(type="reasoning", chunk="thinking"), + llm.parts.StreamEvent(type="text", chunk="hello"), + llm.parts.StreamEvent(type="reasoning", chunk="thinking"), ] mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() assert response.messages()[0].parts == [ - llm.TextPart(text="hello"), - llm.ReasoningPart(text="thinking"), + llm.parts.TextPart(text="hello"), + llm.parts.ReasoningPart(text="thinking"), ] def test_text_tool_call_text_produces_three_parts(self, mock_model): events = [ - llm.StreamEvent(type="text", chunk="before"), - llm.StreamEvent( + llm.parts.StreamEvent(type="text", chunk="before"), + llm.parts.StreamEvent( type="tool_call_name", chunk="search", tool_call_id="c1", ), - llm.StreamEvent( + llm.parts.StreamEvent( type="tool_call_args", chunk='{"q": "x"}', tool_call_id="c1", ), - llm.StreamEvent(type="text", chunk="after"), + llm.parts.StreamEvent(type="text", chunk="after"), ] mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() assert response.messages()[0].parts == [ - llm.TextPart(text="before"), - llm.ToolCallPart(name="search", arguments={"q": "x"}, tool_call_id="c1"), - llm.TextPart(text="after"), + llm.parts.TextPart(text="before"), + llm.parts.ToolCallPart( + name="search", arguments={"q": "x"}, tool_call_id="c1" + ), + llm.parts.TextPart(text="after"), ] def test_tool_call_groups_by_tool_call_id(self, mock_model): events = [ - llm.StreamEvent( + llm.parts.StreamEvent( type="tool_call_name", chunk="search", tool_call_id="c1", ), - llm.StreamEvent( + llm.parts.StreamEvent( type="tool_call_args", chunk='{"q":', tool_call_id="c1", ), - llm.StreamEvent( + llm.parts.StreamEvent( type="tool_call_args", chunk='"weather"}', tool_call_id="c1", @@ -583,7 +589,7 @@ def test_tool_call_groups_by_tool_call_id(self, mock_model): response = mock_model.prompt("hi") response.text() assert response.messages()[0].parts == [ - llm.ToolCallPart( + llm.parts.ToolCallPart( name="search", arguments={"q": "weather"}, tool_call_id="c1", @@ -594,35 +600,47 @@ def test_parallel_tool_calls_interleaved_by_id(self, mock_model): # Two tool calls whose args interleave on the wire — must # still produce two distinct ToolCallParts grouped by id. events = [ - llm.StreamEvent(type="tool_call_name", chunk="search", tool_call_id="A"), - llm.StreamEvent(type="tool_call_name", chunk="lookup", tool_call_id="B"), - llm.StreamEvent(type="tool_call_args", chunk='{"q":"a"}', tool_call_id="A"), - llm.StreamEvent(type="tool_call_args", chunk='{"k":"b"}', tool_call_id="B"), + llm.parts.StreamEvent( + type="tool_call_name", chunk="search", tool_call_id="A" + ), + llm.parts.StreamEvent( + type="tool_call_name", chunk="lookup", tool_call_id="B" + ), + llm.parts.StreamEvent( + type="tool_call_args", chunk='{"q":"a"}', tool_call_id="A" + ), + llm.parts.StreamEvent( + type="tool_call_args", chunk='{"k":"b"}', tool_call_id="B" + ), ] mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() parts = response.messages()[0].parts assert parts == [ - llm.ToolCallPart(name="search", arguments={"q": "a"}, tool_call_id="A"), - llm.ToolCallPart(name="lookup", arguments={"k": "b"}, tool_call_id="B"), + llm.parts.ToolCallPart( + name="search", arguments={"q": "a"}, tool_call_id="A" + ), + llm.parts.ToolCallPart( + name="lookup", arguments={"k": "b"}, tool_call_id="B" + ), ] def test_tool_result_is_always_own_part(self, mock_model): events = [ - llm.StreamEvent( + llm.parts.StreamEvent( type="tool_call_name", chunk="web_search", tool_call_id="c1", server_executed=True, ), - llm.StreamEvent( + llm.parts.StreamEvent( type="tool_call_args", chunk='{"q":"x"}', tool_call_id="c1", server_executed=True, ), - llm.StreamEvent( + llm.parts.StreamEvent( type="tool_result", chunk="results...", tool_call_id="c1", @@ -635,13 +653,13 @@ def test_tool_result_is_always_own_part(self, mock_model): response.text() parts = response.messages()[0].parts assert parts == [ - llm.ToolCallPart( + llm.parts.ToolCallPart( name="web_search", arguments={"q": "x"}, tool_call_id="c1", server_executed=True, ), - llm.ToolResultPart( + llm.parts.ToolResultPart( name="web_search", output="results...", tool_call_id="c1", @@ -653,19 +671,19 @@ def test_two_reasoning_blocks_split_by_tool_call(self, mock_model): # Some providers emit two thinking blocks separated by a tool # call — those should yield two ReasoningParts, not one. events = [ - llm.StreamEvent(type="reasoning", chunk="first"), - llm.StreamEvent(type="tool_call_name", chunk="t", tool_call_id="c1"), - llm.StreamEvent(type="tool_call_args", chunk="{}", tool_call_id="c1"), - llm.StreamEvent(type="reasoning", chunk="second"), + llm.parts.StreamEvent(type="reasoning", chunk="first"), + llm.parts.StreamEvent(type="tool_call_name", chunk="t", tool_call_id="c1"), + llm.parts.StreamEvent(type="tool_call_args", chunk="{}", tool_call_id="c1"), + llm.parts.StreamEvent(type="reasoning", chunk="second"), ] mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() parts = response.messages()[0].parts assert parts == [ - llm.ReasoningPart(text="first"), - llm.ToolCallPart(name="t", arguments={}, tool_call_id="c1"), - llm.ReasoningPart(text="second"), + llm.parts.ReasoningPart(text="first"), + llm.parts.ToolCallPart(name="t", arguments={}, tool_call_id="c1"), + llm.parts.ReasoningPart(text="second"), ] def test_parallel_tool_calls_without_id_each_get_own_part(self, mock_model): @@ -674,36 +692,36 @@ def test_parallel_tool_calls_without_id_each_get_own_part(self, mock_model): # part — otherwise the N tool calls collapse into one with # concatenated names and args. events = [ - llm.StreamEvent(type="tool_call_name", chunk="store_fact"), - llm.StreamEvent(type="tool_call_args", chunk='{"fact":"a"}'), - llm.StreamEvent(type="tool_call_name", chunk="store_fact"), - llm.StreamEvent(type="tool_call_args", chunk='{"fact":"b"}'), - llm.StreamEvent(type="tool_call_name", chunk="store_fact"), - llm.StreamEvent(type="tool_call_args", chunk='{"fact":"c"}'), + llm.parts.StreamEvent(type="tool_call_name", chunk="store_fact"), + llm.parts.StreamEvent(type="tool_call_args", chunk='{"fact":"a"}'), + llm.parts.StreamEvent(type="tool_call_name", chunk="store_fact"), + llm.parts.StreamEvent(type="tool_call_args", chunk='{"fact":"b"}'), + llm.parts.StreamEvent(type="tool_call_name", chunk="store_fact"), + llm.parts.StreamEvent(type="tool_call_args", chunk='{"fact":"c"}'), ] mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() parts = response.messages()[0].parts assert parts == [ - llm.ToolCallPart(name="store_fact", arguments={"fact": "a"}), - llm.ToolCallPart(name="store_fact", arguments={"fact": "b"}), - llm.ToolCallPart(name="store_fact", arguments={"fact": "c"}), + llm.parts.ToolCallPart(name="store_fact", arguments={"fact": "a"}), + llm.parts.ToolCallPart(name="store_fact", arguments={"fact": "b"}), + llm.parts.ToolCallPart(name="store_fact", arguments={"fact": "c"}), ] def test_explicit_part_index_still_works(self, mock_model): # Back-compat: plugins that pass explicit part_index should # behave exactly as before. events = [ - llm.StreamEvent(type="reasoning", chunk="t", part_index=0), - llm.StreamEvent(type="text", chunk="hi", part_index=1), + llm.parts.StreamEvent(type="reasoning", chunk="t", part_index=0), + llm.parts.StreamEvent(type="text", chunk="hi", part_index=1), ] mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() assert response.messages()[0].parts == [ - llm.ReasoningPart(text="t"), - llm.TextPart(text="hi"), + llm.parts.ReasoningPart(text="t"), + llm.parts.TextPart(text="hi"), ] def test_mix_explicit_zero_and_none_for_text_concatenates(self, mock_model): @@ -711,18 +729,18 @@ def test_mix_explicit_zero_and_none_for_text_concatenates(self, mock_model): # plugin pins explicit part_index=0 on the wraparound text # events, and the tool call in between gets None (auto). events = [ - llm.StreamEvent(type="text", chunk="before ", part_index=0), - llm.StreamEvent(type="tool_call_name", chunk="t", tool_call_id="c1"), - llm.StreamEvent(type="tool_call_args", chunk="{}", tool_call_id="c1"), - llm.StreamEvent(type="text", chunk="after", part_index=0), + llm.parts.StreamEvent(type="text", chunk="before ", part_index=0), + llm.parts.StreamEvent(type="tool_call_name", chunk="t", tool_call_id="c1"), + llm.parts.StreamEvent(type="tool_call_args", chunk="{}", tool_call_id="c1"), + llm.parts.StreamEvent(type="text", chunk="after", part_index=0), ] mock_model.enqueue(events) response = mock_model.prompt("hi") response.text() parts = response.messages()[0].parts assert parts == [ - llm.TextPart(text="before after"), - llm.ToolCallPart(name="t", arguments={}, tool_call_id="c1"), + llm.parts.TextPart(text="before after"), + llm.parts.ToolCallPart(name="t", arguments={}, tool_call_id="c1"), ] @@ -731,8 +749,8 @@ class TestStreamEventsLiveDuringStreaming: def test_events_arrive_before_done(self, mock_model): events = [ - llm.StreamEvent(type="reasoning", chunk="t", part_index=0), - llm.StreamEvent(type="text", chunk="hi", part_index=1), + llm.parts.StreamEvent(type="reasoning", chunk="t", part_index=0), + llm.parts.StreamEvent(type="text", chunk="hi", part_index=1), ] mock_model.enqueue(events) response = mock_model.prompt("x") @@ -747,7 +765,9 @@ def test_events_arrive_before_done(self, mock_model): assert response._done def test_stream_events_after_done_replays(self, mock_model): - mock_model.enqueue([llm.StreamEvent(type="text", chunk="hi", part_index=0)]) + mock_model.enqueue( + [llm.parts.StreamEvent(type="text", chunk="hi", part_index=0)] + ) response = mock_model.prompt("x") first = list(response.stream_events()) # Second call replays from the stored events. @@ -770,8 +790,8 @@ class TestAsyncStreamEvents: @pytest.mark.asyncio async def test_async_stream_events_live(self, async_mock_model): events = [ - llm.StreamEvent(type="reasoning", chunk="r", part_index=0), - llm.StreamEvent(type="text", chunk="t", part_index=1), + llm.parts.StreamEvent(type="reasoning", chunk="r", part_index=0), + llm.parts.StreamEvent(type="text", chunk="t", part_index=1), ] async_mock_model.enqueue(events) response = async_mock_model.prompt("x") @@ -783,8 +803,8 @@ async def test_async_stream_events_live(self, async_mock_model): @pytest.mark.asyncio async def test_async_iter_yields_only_text(self, async_mock_model): events = [ - llm.StreamEvent(type="reasoning", chunk="r", part_index=0), - llm.StreamEvent(type="text", chunk="hi", part_index=1), + llm.parts.StreamEvent(type="reasoning", chunk="r", part_index=0), + llm.parts.StreamEvent(type="text", chunk="hi", part_index=1), ] async_mock_model.enqueue(events) response = async_mock_model.prompt("x") @@ -799,7 +819,7 @@ async def test_async_messages_after_await(self, async_mock_model): response = async_mock_model.prompt("x") await response.text() assert await response.messages() == [ - llm.Message(role="assistant", parts=[llm.TextPart(text="hi")]) + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="hi")]) ] @@ -814,7 +834,7 @@ def test_sync_messages_is_callable_and_returns_list(self, mock_model): # No prior .text() or iteration — calling messages() forces # execution and returns the assembled list. assert response.messages() == [ - llm.Message(role="assistant", parts=[llm.TextPart(text="hi")]) + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="hi")]) ] def test_sync_messages_after_text_returns_same_list(self, mock_model): @@ -822,7 +842,7 @@ def test_sync_messages_after_text_returns_same_list(self, mock_model): response = mock_model.prompt("x") response.text() assert response.messages() == [ - llm.Message(role="assistant", parts=[llm.TextPart(text="hi")]) + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="hi")]) ] @pytest.mark.asyncio @@ -832,7 +852,7 @@ async def test_async_messages_is_awaitable(self, async_mock_model): # No prior await — `await response.messages()` forces it. result = await response.messages() assert result == [ - llm.Message(role="assistant", parts=[llm.TextPart(text="hi")]) + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="hi")]) ] @pytest.mark.asyncio @@ -842,7 +862,7 @@ async def test_async_messages_after_text_returns_same_list(self, async_mock_mode await response.text() result = await response.messages() assert result == [ - llm.Message(role="assistant", parts=[llm.TextPart(text="hi")]) + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="hi")]) ] @@ -860,15 +880,17 @@ def test_prompt_text_synthesizes_user_message(self, mock_model): from llm.models import Prompt p = Prompt("hi", model=mock_model) - assert p.messages == [llm.Message(role="user", parts=[llm.TextPart(text="hi")])] + assert p.messages == [ + llm.Message(role="user", parts=[llm.parts.TextPart(text="hi")]) + ] def test_system_and_prompt_synthesizes_two_messages(self, mock_model): from llm.models import Prompt p = Prompt("hi", model=mock_model, system="be brief") assert p.messages == [ - llm.Message(role="system", parts=[llm.TextPart(text="be brief")]), - llm.Message(role="user", parts=[llm.TextPart(text="hi")]), + llm.Message(role="system", parts=[llm.parts.TextPart(text="be brief")]), + llm.Message(role="user", parts=[llm.parts.TextPart(text="hi")]), ] def test_attachments_join_user_message(self, mock_model): @@ -880,8 +902,8 @@ def test_attachments_join_user_message(self, mock_model): llm.Message( role="user", parts=[ - llm.TextPart(text="look"), - llm.AttachmentPart(attachment=att), + llm.parts.TextPart(text="look"), + llm.parts.AttachmentPart(attachment=att), ], ) ] @@ -895,7 +917,9 @@ def test_tool_results_become_tool_role_message(self, mock_model): assert p.messages == [ llm.Message( role="tool", - parts=[llm.ToolResultPart(name="t", output="ok", tool_call_id="c1")], + parts=[ + llm.parts.ToolResultPart(name="t", output="ok", tool_call_id="c1") + ], ) ] @@ -1044,8 +1068,10 @@ def test_conversation_preserves_reasoning_and_tool_call_parts(self, mock_model): thinking (Claude) and tool-use round-trips.""" mock_model.enqueue( [ - llm.StreamEvent(type="reasoning", chunk="thinking...", part_index=0), - llm.StreamEvent(type="text", chunk="answer", part_index=1), + llm.parts.StreamEvent( + type="reasoning", chunk="thinking...", part_index=0 + ), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), ] ) mock_model.enqueue(["follow-up answer"]) @@ -1060,8 +1086,8 @@ def test_conversation_preserves_reasoning_and_tool_call_parts(self, mock_model): llm.Message( role="assistant", parts=[ - llm.ReasoningPart(text="thinking..."), - llm.TextPart(text="answer"), + llm.parts.ReasoningPart(text="thinking..."), + llm.parts.TextPart(text="answer"), ], ), llm.user("q2"), @@ -1112,7 +1138,9 @@ def test_from_row_response_messages_synthesized_from_chunks( # to synthesizing a TextPart. assert rehydrated._stream_events == [] assert rehydrated.messages() == [ - llm.Message(role="assistant", parts=[llm.TextPart(text="answer text")]) + llm.Message( + role="assistant", parts=[llm.parts.TextPart(text="answer text")] + ) ] def test_llm_dash_c_chain_preserves_prior_assistant_turn( @@ -1244,12 +1272,12 @@ class ToolCallMock(type(mock_model)): def execute(self, prompt, stream, response, conversation): # Yield the assistant turn's parts as StreamEvents so # response.messages() contains the tool call. - yield llm.StreamEvent( + yield llm.parts.StreamEvent( type="tool_call_name", chunk="echo", tool_call_id="c1", ) - yield llm.StreamEvent( + yield llm.parts.StreamEvent( type="tool_call_args", chunk='{"x": 1}', tool_call_id="c1", @@ -1282,12 +1310,12 @@ class ToolCallMock(type(mock_model)): supports_tools = True def execute(self, prompt, stream, response, conversation): - yield llm.StreamEvent( + yield llm.parts.StreamEvent( type="tool_call_name", chunk="echo", tool_call_id="c1", ) - yield llm.StreamEvent( + yield llm.parts.StreamEvent( type="tool_call_args", chunk='{"x": 1}', tool_call_id="c1", @@ -1329,10 +1357,10 @@ def execute(self, prompt, stream, response, conversation): response.add_tool_call( llm.ToolCall(name="echo", arguments={"x": 42}, tool_call_id="c1") ) - yield llm.StreamEvent( + yield llm.parts.StreamEvent( type="tool_call_name", chunk="echo", tool_call_id="c1" ) - yield llm.StreamEvent( + yield llm.parts.StreamEvent( type="tool_call_args", chunk='{"x": 42}', tool_call_id="c1", @@ -1372,10 +1400,10 @@ def execute(self, prompt, stream, response, conversation): response.add_tool_call( llm.ToolCall(name="echo", arguments={"x": 1}, tool_call_id="c1") ) - yield llm.StreamEvent( + yield llm.parts.StreamEvent( type="tool_call_name", chunk="echo", tool_call_id="c1" ) - yield llm.StreamEvent( + yield llm.parts.StreamEvent( type="tool_call_args", chunk='{"x": 1}', tool_call_id="c1", @@ -1405,10 +1433,10 @@ class ToolCallMock(type(mock_model)): supports_tools = True def execute(self, prompt, stream, response, conversation): - yield llm.StreamEvent( + yield llm.parts.StreamEvent( type="tool_call_name", chunk="echo", tool_call_id="c1" ) - yield llm.StreamEvent( + yield llm.parts.StreamEvent( type="tool_call_args", chunk='{"x": 1}', tool_call_id="c1", @@ -1461,10 +1489,10 @@ async def execute(self, prompt, stream, response, conversation): response.add_tool_call( llm.ToolCall(name="echo", arguments={"x": 7}, tool_call_id="c1") ) - yield llm.StreamEvent( + yield llm.parts.StreamEvent( type="tool_call_name", chunk="echo", tool_call_id="c1" ) - yield llm.StreamEvent( + yield llm.parts.StreamEvent( type="tool_call_args", chunk='{"x": 7}', tool_call_id="c1", @@ -1494,12 +1522,12 @@ class ToolCallMock(type(async_mock_model)): supports_tools = True async def execute(self, prompt, stream, response, conversation): - yield llm.StreamEvent( + yield llm.parts.StreamEvent( type="tool_call_name", chunk="echo", tool_call_id="c1", ) - yield llm.StreamEvent( + yield llm.parts.StreamEvent( type="tool_call_args", chunk='{"x": 1}', tool_call_id="c1", @@ -1750,13 +1778,13 @@ def test_from_dict_then_reply_continues_conversation(self, mock_model): def test_to_dict_preserves_reasoning_and_signatures(self, mock_model): mock_model.enqueue( [ - llm.StreamEvent( + llm.parts.StreamEvent( type="reasoning", chunk="thinking...", part_index=0, provider_metadata={"anthropic": {"signature": "sig-abc"}}, ), - llm.StreamEvent(type="text", chunk="answer", part_index=1), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), ] ) r = mock_model.prompt("q") @@ -1767,7 +1795,7 @@ def test_to_dict_preserves_reasoning_and_signatures(self, mock_model): msgs = restored.messages() assert msgs[0].role == "assistant" - assert isinstance(msgs[0].parts[0], llm.ReasoningPart) + assert isinstance(msgs[0].parts[0], llm.parts.ReasoningPart) assert msgs[0].parts[0].text == "thinking..." assert msgs[0].parts[0].provider_metadata == { "anthropic": {"signature": "sig-abc"} @@ -1778,13 +1806,13 @@ def test_from_dict_reply_includes_prior_reasoning_in_chain(self, mock_model): back to the model for multi-turn extended thinking.""" mock_model.enqueue( [ - llm.StreamEvent( + llm.parts.StreamEvent( type="reasoning", chunk="thinking...", part_index=0, provider_metadata={"anthropic": {"signature": "sig-xyz"}}, ), - llm.StreamEvent(type="text", chunk="answer", part_index=1), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), ] ) mock_model.enqueue(["a2"]) @@ -1799,7 +1827,7 @@ def test_from_dict_reply_includes_prior_reasoning_in_chain(self, mock_model): # The signature must be in the chain sent to the model. chain = r2.prompt.messages reasoning_parts = [ - p for m in chain for p in m.parts if isinstance(p, llm.ReasoningPart) + p for m in chain for p in m.parts if isinstance(p, llm.parts.ReasoningPart) ] assert len(reasoning_parts) == 1 assert reasoning_parts[0].provider_metadata == { @@ -1832,7 +1860,9 @@ class TestChainResponseStreamEvents: def test_sync_chain_stream_events_yields_text_when_no_tools(self, mock_model): # Chain with no tool calls is a single-response chain — its # stream_events should concatenate from each underlying response. - mock_model.enqueue([llm.StreamEvent(type="text", chunk="done", part_index=0)]) + mock_model.enqueue( + [llm.parts.StreamEvent(type="text", chunk="done", part_index=0)] + ) chain = mock_model.conversation().chain("q") events = list(chain.stream_events()) assert [e.type for e in events] == ["text"] @@ -1841,7 +1871,7 @@ def test_sync_chain_stream_events_yields_text_when_no_tools(self, mock_model): @pytest.mark.asyncio async def test_async_chain_astream_events_yields(self, async_mock_model): async_mock_model.enqueue( - [llm.StreamEvent(type="text", chunk="done", part_index=0)] + [llm.parts.StreamEvent(type="text", chunk="done", part_index=0)] ) chain = async_mock_model.conversation().chain("q") events = [] @@ -1901,14 +1931,14 @@ def test_roundtrip_preserves_tool_calls_and_results(self, mock_model): llm.user("what's the weather?"), llm.assistant( "let me check", - llm.ToolCallPart( + llm.parts.ToolCallPart( name="get_weather", arguments={"city": "Paris"}, tool_call_id="c1", ), ), llm.tool_message( - llm.ToolResultPart( + llm.parts.ToolResultPart( name="get_weather", output="sunny", tool_call_id="c1", @@ -1926,8 +1956,8 @@ def test_roundtrip_preserves_redacted_reasoning(self, mock_model): msg = llm.Message( role="assistant", parts=[ - llm.ReasoningPart(text="", redacted=True), - llm.TextPart(text="result"), + llm.parts.ReasoningPart(text="", redacted=True), + llm.parts.TextPart(text="result"), ], ) restored = llm.Message.from_dict(json.loads(json.dumps(msg.to_dict()))) @@ -1937,11 +1967,11 @@ def test_roundtrip_preserves_provider_metadata(self, mock_model): msg = llm.Message( role="assistant", parts=[ - llm.ReasoningPart( + llm.parts.ReasoningPart( text="thinking", provider_metadata={"anthropic": {"signature": "abc"}}, ), - llm.TextPart(text="answer"), + llm.parts.TextPart(text="answer"), ], ) restored = llm.Message.from_dict(json.loads(json.dumps(msg.to_dict()))) diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 651608823..744499ee2 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -79,69 +79,71 @@ def _adapter(self, td): return TypeAdapter(td) def test_text_part_matches(self): - d = llm.TextPart(text="hello").to_dict() + d = llm.parts.TextPart(text="hello").to_dict() self._adapter(TextPartDict).validate_python(d) def test_text_part_with_provider_metadata_matches(self): - d = llm.TextPart( + d = llm.parts.TextPart( text="hi", provider_metadata={"anthropic": {"cached": True}} ).to_dict() self._adapter(TextPartDict).validate_python(d) def test_reasoning_part_redacted_matches(self): - d = llm.ReasoningPart(text="", redacted=True).to_dict() + d = llm.parts.ReasoningPart(text="", redacted=True).to_dict() self._adapter(ReasoningPartDict).validate_python(d) def test_reasoning_part_with_signature_matches(self): - d = llm.ReasoningPart( + d = llm.parts.ReasoningPart( text="thinking...", provider_metadata={"anthropic": {"signature": "sig-abc"}}, ).to_dict() self._adapter(ReasoningPartDict).validate_python(d) def test_tool_call_part_matches(self): - d = llm.ToolCallPart( + d = llm.parts.ToolCallPart( name="search", arguments={"q": "x"}, tool_call_id="c1" ).to_dict() self._adapter(ToolCallPartDict).validate_python(d) def test_tool_result_part_matches(self): - d = llm.ToolResultPart( + d = llm.parts.ToolResultPart( name="search", output="result", tool_call_id="c1" ).to_dict() self._adapter(ToolResultPartDict).validate_python(d) def test_attachment_part_with_url_matches(self): att = llm.Attachment(type="image/jpeg", url="https://example.com/cat.jpg") - d = llm.AttachmentPart(attachment=att).to_dict() + d = llm.parts.AttachmentPart(attachment=att).to_dict() self._adapter(AttachmentPartDict).validate_python(d) def test_attachment_part_with_bytes_matches(self): att = llm.Attachment(type="image/png", content=b"\x89PNG...") - d = llm.AttachmentPart(attachment=att).to_dict() + d = llm.parts.AttachmentPart(attachment=att).to_dict() self._adapter(AttachmentPartDict).validate_python(d) class TestPartDiscriminatedUnion: def test_text_part_validates_as_part_dict(self): - d = llm.TextPart(text="hi").to_dict() + d = llm.parts.TextPart(text="hi").to_dict() TypeAdapter(PartDict).validate_python(d) def test_reasoning_part_validates_as_part_dict(self): - d = llm.ReasoningPart(text="thinking").to_dict() + d = llm.parts.ReasoningPart(text="thinking").to_dict() TypeAdapter(PartDict).validate_python(d) def test_tool_call_part_validates_as_part_dict(self): - d = llm.ToolCallPart(name="t", arguments={}, tool_call_id="c1").to_dict() + d = llm.parts.ToolCallPart(name="t", arguments={}, tool_call_id="c1").to_dict() TypeAdapter(PartDict).validate_python(d) def test_tool_result_part_validates_as_part_dict(self): - d = llm.ToolResultPart(name="t", output="out", tool_call_id="c1").to_dict() + d = llm.parts.ToolResultPart( + name="t", output="out", tool_call_id="c1" + ).to_dict() TypeAdapter(PartDict).validate_python(d) def test_attachment_part_validates_as_part_dict(self): att = llm.Attachment(type="image/jpeg", url="http://x") - d = llm.AttachmentPart(attachment=att).to_dict() + d = llm.parts.AttachmentPart(attachment=att).to_dict() TypeAdapter(PartDict).validate_python(d) def test_unknown_type_rejected(self): @@ -158,12 +160,12 @@ def test_assistant_with_mixed_parts_matches(self): m = llm.Message( role="assistant", parts=[ - llm.ReasoningPart( + llm.parts.ReasoningPart( text="thinking", provider_metadata={"anthropic": {"signature": "s"}}, ), - llm.TextPart(text="answer"), - llm.ToolCallPart( + llm.parts.TextPart(text="answer"), + llm.parts.ToolCallPart( name="search", arguments={"q": "x"}, tool_call_id="c1", @@ -174,7 +176,7 @@ def test_assistant_with_mixed_parts_matches(self): def test_tool_role_message_with_results_matches(self): m = llm.tool_message( - llm.ToolResultPart(name="s", output="r", tool_call_id="c1"), + llm.parts.ToolResultPart(name="s", output="r", tool_call_id="c1"), ) TypeAdapter(MessageDict).validate_python(m.to_dict()) @@ -191,13 +193,13 @@ def test_mock_response_to_dict_matches(self, mock_model): def test_response_with_reasoning_matches(self, mock_model): mock_model.enqueue( [ - llm.StreamEvent( + llm.parts.StreamEvent( type="reasoning", chunk="thinking", part_index=0, provider_metadata={"anthropic": {"signature": "s"}}, ), - llm.StreamEvent(type="text", chunk="answer", part_index=1), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), ] ) r = mock_model.prompt("q") @@ -264,31 +266,31 @@ class TestAnnotations: def test_text_part_to_dict_annotation(self): import typing - hints = typing.get_type_hints(llm.TextPart.to_dict) + hints = typing.get_type_hints(llm.parts.TextPart.to_dict) assert hints["return"] is TextPartDict def test_reasoning_part_to_dict_annotation(self): import typing - hints = typing.get_type_hints(llm.ReasoningPart.to_dict) + hints = typing.get_type_hints(llm.parts.ReasoningPart.to_dict) assert hints["return"] is ReasoningPartDict def test_tool_call_part_to_dict_annotation(self): import typing - hints = typing.get_type_hints(llm.ToolCallPart.to_dict) + hints = typing.get_type_hints(llm.parts.ToolCallPart.to_dict) assert hints["return"] is ToolCallPartDict def test_tool_result_part_to_dict_annotation(self): import typing - hints = typing.get_type_hints(llm.ToolResultPart.to_dict) + hints = typing.get_type_hints(llm.parts.ToolResultPart.to_dict) assert hints["return"] is ToolResultPartDict def test_attachment_part_to_dict_annotation(self): import typing - hints = typing.get_type_hints(llm.AttachmentPart.to_dict) + hints = typing.get_type_hints(llm.parts.AttachmentPart.to_dict) assert hints["return"] is AttachmentPartDict def test_message_to_dict_annotation(self): From 35c7533c185e8f0aaf0cb2639652a19d0382bf80 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 16:49:05 -0700 Subject: [PATCH 063/258] Edited the changelog for 0.32a0 --- docs/changelog.md | 41 ++++++++++++++--------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 8ffe9d319..834d50b39 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,44 +3,31 @@ (v0_32_a0)= ## 0.32a0 (2026-04-28) -This alpha introduces a new structured representation for LLM conversations. Prompt inputs and response outputs can now be expressed as a list of `Message` objects, each containing typed `Part` objects (text, reasoning, tool calls, tool results, attachments). The previous string-based API continues to work unchanged. +This alpha introduces a major backwards-compatible refactor. Models can now be prompted with a list of messages, OpenAI Chat Completions style, and the response can now be iterated over as a sequence of mixed types of content, for example reasoning tokens mixed with text tokens mixed with tool calls. -Plugin authors should read the substantially expanded {ref}`Advanced model plugins ` documentation, which now covers `StreamEvent`, consuming `prompt.messages`, and round-tripping opaque provider metadata such as Anthropic extended-thinking signatures and Gemini `thoughtSignature` values. +Prompt inputs and response outputs are now expressed as a list of `Message` objects, each containing typed `Part` objects (text, reasoning, tool calls, tool results, attachments). + +The `llm` CLI tool can now display reasoning tokens while executing a prompt. + +Plugin authors should read the expanded {ref}`Advanced model plugins ` documentation, which now covers `StreamEvent`, consuming `prompt.messages`, and round-tripping opaque provider metadata such as Anthropic extended-thinking signatures and Gemini `thoughtSignature` values. ### Structured messages and streaming events -- New value types exported from the top-level `llm` package: `Message`, `Part`, `TextPart`, `ReasoningPart`, `ToolCallPart`, `ToolResultPart`, `AttachmentPart`, and `StreamEvent`. Plus constructor helpers `llm.user()`, `llm.assistant()`, `llm.system()`, and `llm.tool_message()` that accept strings, `Attachment` instances, or nested `Part` lists. -- New `messages=` keyword argument on `model.prompt()`, `conversation.prompt()`, `model.chain()`, `conversation.chain()`, and their async counterparts. When provided, `messages=` is the authoritative input to the model — `prompt.messages` returns it verbatim. The legacy `prompt=`, `system=`, `attachments=`, and `tool_results=` keywords still work and synthesize into the same `Message` list internally. -- New `response.stream_events()` and `response.astream_events()` methods yielding typed `StreamEvent` objects (`type` is one of `"text"`, `"reasoning"`, `"tool_call_name"`, `"tool_call_args"`, `"tool_result"`, plus a `redacted=True` marker for opaque reasoning). Each event carries a `part_index` that groups events into the same logical Part. Iterating against `response` directly continues to yield only text strings. -- New `response.messages()` method (async: `await response.messages()`) returning the assembled `list[Message]` produced by the model. Calling it forces execution if the response has not yet been drained, matching the `.text()` / `.json()` / `.tool_calls()` pattern. +- New `llm.Message` value type and constructor helpers `llm.user()`, `llm.assistant()`, `llm.system()`, and `llm.tool_message()` for building structured prompt inputs. The helpers accept strings, `Attachment` instances, or nested `Part` lists. +- New `messages=` keyword argument on `model.prompt()`, `conversation.prompt()`, `model.chain()`, `conversation.chain()`, and their async counterparts. The `prompt=`, `system=`, `attachments=`, and `tool_results=` keywords still work and synthesize into the same `Message` list internally. +- New `response.stream_events()` and `response.astream_events()` methods yielding typed `StreamEvent` objects (`type` is one of `"text"`, `"reasoning"`, `"tool_call_name"`, `"tool_call_args"`, `"tool_result"`, plus a `redacted=True` marker for opaque reasoning). Iterating against `response` directly continues to yield only text strings. +- New `response.messages()` method (async: `await response.messages()`) returning the assembled `list[Message]` produced by the model. Calling it forces execution if the response prompt has not yet been executed. - New `response.reply(prompt=None, **kwargs)` method that continues the conversation from any `Response`, regardless of origin. When the previous response made tool calls and `tool_results=` was not passed, `reply()` automatically executes the pending tool calls and threads the results into the next turn. On async responses `reply()` is awaitable. - New `response.to_dict()` and `Response.from_dict(data, *, model=None)` for JSON-safe serialization of a full conversation turn — model id, input chain, assembled output (including reasoning parts and provider metadata), options, and audit fields. Reasoning signatures and `thoughtSignature` values round-trip via `provider_metadata`, so multi-turn extended thinking works across process boundaries. -- New `llm/serialization.py` module exposing `MessageDict`, `PartDict`, `ResponseDict`, `PromptDict`, `UsageDict`, `AttachmentDict`, and the per-Part TypedDicts. Every `to_dict()` / `from_dict()` method is annotated with the matching TypedDict, so consumers get autocomplete, mypy coverage, and pydantic `TypeAdapter` validation for free. +- New `llm/serialization.py` module exposing `MessageDict`, `PartDict`, `ResponseDict`, `PromptDict`, `UsageDict`, `AttachmentDict`, and the per-Part TypedDicts. Every `to_dict()` / `from_dict()` method is annotated with the matching TypedDict. - `Response.prompt.messages` is now the canonical structured input across the entire conversation chain. `Conversation.prompt` and `AsyncConversation.prompt` pre-compute the full chain (prior input + prior output + new turn) before constructing the next `Prompt`, so `response.prompt.messages` is always exactly what the model was sent. ### CLI -- `llm prompt` and `llm chat` now display visible reasoning text to stderr in a dim style while the response streams, with a newline inserted at the reasoning-to-text transition so the assistant's final answer starts on a fresh line. +- `llm prompt` and `llm chat` now display visible reasoning text to stderr in a dim style while the response streams. - New `-R/--no-reasoning` flag for `llm prompt` and `llm chat` to suppress the reasoning stream. -- `llm logs --md` now renders any visible reasoning emitted during a response under a `## Reasoning` heading above the response. -- New `reasoning` column on the `responses` table populated from the visible-reasoning text. Added by migration `m022`. - -### OpenAI plugin - -- Rewritten to consume `prompt.messages` and dispatch per-`Part` into OpenAI's wire format. The legacy `prompt=` / `system=` / `attachments=` / `tool_results=` entry points continue to work because `Prompt.messages` synthesizes those into the same `Message` list the new code reads. -- `Chat.execute()` and `AsyncChat.execute()` yield `StreamEvent` objects (text, tool-call name, tool-call argument deltas) for both streaming and non-streaming paths. Empty-string content deltas (OpenAI's first `role=assistant` chunk) are now skipped as noise. -- Reasoning token counts from OpenAI reasoning models (e.g. GPT-5 family) are captured before `set_usage` mutates the usage dict and surfaced as a redacted `ReasoningPart` on the assembled message, so callers can render "the model used N reasoning tokens here" without any visible reasoning text being available. - -### Documentation - -- Major expansion of {ref}`Advanced model plugins ` covering the `StreamEvent` contract, `part_index` allocation rules, server-executed tools, opaque `provider_metadata`, the new `prompt.messages` invariant for `build_messages`, role mapping across OpenAI / Anthropic / Gemini conventions, and how to fold reasoning signatures and `thoughtSignature` values back into outgoing requests. -- New {ref}`Structured messages and streaming events ` section in the Python API docs walking through `messages=[...]`, `stream_events()`, `response.messages()`, `response.reply()`, and the `to_dict()` / `from_dict()` round-trip pattern. - -### Bug fixes - -- Parallel tool calls emitted without a `tool_call_id` (e.g. Gemini's parallel function calls) are no longer collapsed into a single `ToolCallPart` with concatenated names and arguments. Each fresh `tool_call_name` event now allocates a new `part_index`, producing one `ToolCallPart` per call. -- Tool-result turns inside a `chain()` loop now carry `system=` and `system_fragments=` forward from the initial prompt. Adapters that read `prompt.system` directly (e.g. OpenAI's Chat completions, which sends system as its own message) previously saw an empty system on every turn after the first. -- Fixed a regression where `llm -c` (and `load_conversation().prompt(...)`) dropped the prior assistant turn from the chain when the response was rehydrated from SQLite. `response.messages` now falls back to synthesizing from the persisted text and tool-call columns when no `StreamEvent`s are available. +- `llm logs` now renders any visible reasoning emitted during a response under a `## Reasoning` heading above the response. +- New `reasoning` column on the `responses` table populated from the visible-reasoning text. (v0_31)= ## 0.31 (2026-04-24) From 5789bc9e36d2be48f9588d548914d5f3bfd85c84 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 16:59:11 -0700 Subject: [PATCH 064/258] It's actually the 0.32 alpha --- docs/plugins/advanced-model-plugins.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index cdae9d087..3c74d01d4 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -252,7 +252,7 @@ Conversation history — including attachments from prior turns — is available ## Structured messages and streaming events -The 0.31 alpha introduced a richer contract for plugins than "yield strings": +The 0.32 alpha introduced a richer contract for plugins than "yield strings": 1. **`execute()` yields `StreamEvent` objects** (or plain `str`, still supported) so text, reasoning (thinking tokens), tool calls, and server-side tool results each surface as their own event type. The framework assembles these into typed `Part` objects. 2. **`build_messages` (or equivalent) reads `prompt.messages`** — a `list[llm.Message]` that is the complete input chain for this turn. From 02c9af048f00e0c3a5db40da5ea5305e71ce1618 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 17:07:51 -0700 Subject: [PATCH 065/258] A bunch of documentation edits --- docs/plugins/advanced-model-plugins.md | 67 ++++++-------------------- 1 file changed, 14 insertions(+), 53 deletions(-) diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index 3c74d01d4..8d0027d96 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -258,7 +258,7 @@ The 0.32 alpha introduced a richer contract for plugins than "yield strings": 2. **`build_messages` (or equivalent) reads `prompt.messages`** — a `list[llm.Message]` that is the complete input chain for this turn. 3. **Opaque provider tokens round-trip via `provider_metadata`** — Anthropic thinking signatures, Gemini thought signatures, OpenAI Responses API encrypted reasoning blobs. Plugins stash whatever the API returns, then echo it back on the next request. -**Backward compatibility is guaranteed.** A plugin that still yields plain `str` from `execute()` works unchanged — each string is wrapped as a `StreamEvent(type="text", chunk=...)` internally. +**Older plugins still work.** A plugin that still yields plain `str` from `execute()` works unchanged — each string is wrapped as a `StreamEvent(type="text", chunk=...)` internally. ### Yielding StreamEvent from execute() @@ -276,8 +276,6 @@ def execute(self, prompt, stream, response, conversation, key=None): yield StreamEvent(type="reasoning", chunk=chunk.text) ``` -That's the whole pattern for most plugins. The framework figures out which events group into which Part. - A `StreamEvent` has four frequently-used fields: - **`type`** — one of `"text"`, `"reasoning"`, `"tool_call_name"`, `"tool_call_args"`, `"tool_result"`. @@ -287,7 +285,7 @@ A `StreamEvent` has four frequently-used fields: Three additional fields exist for special cases: -- **`server_executed: bool`** — set `True` for server-side tool calls (for example, Anthropic web search) and their results. The model ran the tool internally. +- **`server_executed: bool`** — set `True` for server-side tool calls (for example, Anthropic web search) and their results. This means the model ran the tool internally as part of responding to the prompt. - **`tool_name`** — set on `tool_result` events to identify which tool this result came from. - **`part_index: int | None`** — defaults to `None`, which means "let the framework decide which Part this event belongs to." Pass an explicit integer only when you need to override the default grouping (see [below](#part-index-overrides)). @@ -321,27 +319,13 @@ You can mix explicit indices with `None` in the same stream — the framework re ### Reasoning tokens -Two modes are supported: - -**Streamed reasoning text** (Anthropic extended thinking, Gemini with `includeThoughts: true`): - -```python -yield StreamEvent(type="reasoning", chunk=thinking_chunk) -``` - -Reasoning events that appear before/after text events become distinct `ReasoningPart` and `TextPart` entries in `response.messages` automatically. If your provider emits two thinking blocks separated by a tool call, you'll get two `ReasoningPart`s — exactly what you want. - -**Opaque reasoning token count** (OpenAI o-series, Gemini without `includeThoughts`): - -The provider reports only a count — no reasoning text. Record the count on the Response object and the framework will prepend a redacted `ReasoningPart`: +For streamed reasoning text: ```python -# Anywhere before set_usage runs (usually at the end of execute): -if reasoning_tokens > 0: - response._reasoning_token_count = reasoning_tokens +yield StreamEvent(type="reasoning", chunk=text_chunk) ``` -For OpenAI this count lives in `usage.completion_tokens_details.reasoning_tokens`; read it **before** calling `self.set_usage()` — `set_usage()` mutates the usage dict via `pop()` and simplifies out zero-valued entries. +Reasoning events that appear before/after text events become distinct `ReasoningPart` and `TextPart` entries in `response.messages` automatically. If your provider emits two thinking blocks separated by a tool call, you'll get two `ReasoningPart`s. ### Tool calls @@ -361,9 +345,9 @@ yield StreamEvent( ) ``` -The framework groups them by `tool_call_id` — so parallel tool calls (where args for tool A and tool B interleave on the wire) just work without any per-call index tracking. Some providers (Gemini) emit the complete tool call in one chunk — fine; emit both events back-to-back with the full name and full JSON. +The framework groups them by `tool_call_id` — so parallel tool calls (where args for tool A and tool B interleave on the wire) work without any per-call index tracking. Some providers (Gemini) emit the complete tool call in one chunk — it's OK to emit both events back-to-back with the full name and full JSON. -For client-side tool calls — tools that LLM should execute locally in a chain — **also call `response.add_tool_call()`**. The chain-execution path (`response.tool_calls()` → `execute_tool_calls()`) reads from the explicitly-added list, not from the StreamEvent buffer. Your code should do both: +For client-side tool calls — tools that LLM should execute locally in a chain — **also call `response.add_tool_call()`**. The chain-execution path (`response.tool_calls()` → `execute_tool_calls()`) reads from the explicitly-added list, not from the StreamEvent buffer. ```python response.add_tool_call( @@ -377,7 +361,7 @@ response.add_tool_call( ### Server-side tool calls -For tools the API executes internally, set `server_executed=True` on the events. Anthropic web search is a good concrete example: the API returns a `server_tool_use` block for the search request, followed by a `web_search_tool_result` block containing the result payload. +For tools the API executes internally, set `server_executed=True` on the events. Anthropic web search is an example: the API returns a `server_tool_use` block for the search request, followed by a `web_search_tool_result` block containing the result payload. ```python yield StreamEvent( @@ -407,9 +391,9 @@ yield StreamEvent( ) ``` -For providers that don't stream server-tool-result contents (Anthropic's `web_search_tool_result` blocks only arrive in the final message), do the emission as a post-stream step — after the main iteration loop completes, inspect the final message and emit tool_result events for any server-side results. +For providers that don't stream server-tool-result contents (Anthropic's `web_search_tool_result` blocks only arrive in the final message), emit those results as a post-stream step. After the main iteration loop completes, inspect the final message and emit tool_result events for any server-side results. -Do **not** call `response.add_tool_call()` for server-side tool calls unless you intentionally want LLM to run a separate local tool too. The provider has already executed these calls; represent them as `StreamEvent`s so they are preserved in `response.messages` and can be replayed in future turns. +Do **not** call `response.add_tool_call()` for server-side tool calls. This method should only be used for tool calls that need to be executed locally by the framework. ### Opaque provider metadata @@ -442,11 +426,11 @@ yield StreamEvent( ) ``` -Treat other providers' entries as opaque; don't parse them. The framework round-trips the value verbatim via JSON, so use JSON-safe primitives (string, int, bool, dict, list) — avoid custom classes or bytes (base64-encode bytes if you need them). +The framework round-trips the value verbatim via JSON, so use JSON-safe primitives (string, int, bool, dict, list) for provider metadata - use base64 encoding if you need to store binary data. ### Non-streaming path -When `stream=False` (or the provider returns a complete message at once), emit one event per content block — no index tracking required: +When `stream=False` (or the provider returns a complete message at once), emit one event per content block. ```python else: @@ -476,9 +460,9 @@ else: ## Consuming prompt.messages in build_messages -`prompt.messages` is an `list[llm.Message]` that is always **the complete input chain for this turn** — whether the caller supplied it explicitly via `model.prompt(messages=[...])`, or it was synthesized from legacy kwargs (`prompt=`, `system=`, `attachments=`, `tool_results=`), or it was pre-built by a `Conversation` or by `response.reply()`. +`prompt.messages` is an `list[llm.Message]` that is always **the complete input chain for this turn** — whether the caller supplied it explicitly via `model.prompt(messages=[...])`, or it was synthesized from kwargs (`prompt=`, `system=`, `attachments=`, `tool_results=`), or it was pre-built by a `Conversation` or by `response.reply()`. -**Do not also walk `conversation.responses`.** Under the invariant, history is already baked into `prompt.messages`; walking the conversation would double-emit. +**Do not also walk `conversation.responses`.** History is already baked into `prompt.messages`; walking the conversation would double-emit. A plugin's `build_messages` (or equivalent) iterates `prompt.messages` and dispatches per `Part` subtype: @@ -542,29 +526,6 @@ def _append_message(self, out, msg): out.append({"role": role, "content": parts}) ``` -### Role mapping - -LLM uses four roles: `"user"`, `"assistant"`, `"system"`, `"tool"`. Providers differ: - -- **OpenAI Chat Completions** — carries system in the messages array. `"tool"` → `{"role": "tool", "tool_call_id": ..., "content": ...}` per result. -- **Anthropic Messages** — system on a separate `system=` kwarg. `"tool"` → user-role message with `tool_result` blocks. `"assistant"` unchanged. -- **Gemini Generate Content** — system on `systemInstruction`. `"assistant"` → `"model"`. `"tool"` → user-role with `function_response` parts. - -For adapters that need system separately: filter `msg.role == "system"` out of the messages loop and read the current-turn system from `prompt.system` (the synthesized string of `prompt._system` + any system_fragments). The `prompt.system` attribute remains populated by Conversation.prompt for backward compatibility. - -### Role-alternation merging - -Several providers require strict alternation between user and assistant (or equivalent) messages. When two consecutive `llm.Message` values map to the same provider-side role, merge their parts into one provider message: - -```python -if out and out[-1]["role"] == role: - out[-1]["content"].extend(parts) -else: - out.append({"role": role, "content": parts}) -``` - -This is especially relevant for `tool` + `user` — both typically map to a `user` turn for Anthropic/Gemini. - ## Restoring opaque metadata on subsequent requests When a conversation continues, your `build_messages` walks prior-turn Parts via `prompt.messages`. Each Part's `provider_metadata` is a `dict[str, dict]` keyed by provider name — extract your namespace and fold the fields back into the outgoing request body: From 838d5575e6388d44cda600ebbf87f141b77850ac Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 17:45:55 -0700 Subject: [PATCH 066/258] Test to_dict() does not emit keys absent from the TypedDict --- tests/test_serialization.py | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 744499ee2..3cc1c868c 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -325,3 +325,82 @@ def test_json_roundtrip_validates(self, mock_model): parsed = json.loads(payload) # Parsed dict should still conform to ResponseDict. TypeAdapter(ResponseDict).validate_python(parsed) + + +# ---- to_dict() must not emit keys absent from the TypedDict -------- +# +# pydantic's TypeAdapter on a TypedDict silently drops keys that aren't +# declared, so the round-trip tests above will not catch the case where +# .to_dict() starts emitting a brand-new key that nobody added to the +# TypedDict. These tests close that gap by asserting the set of keys +# .to_dict() returns is a subset of the union of required + optional +# keys declared on the corresponding TypedDict. + + +def _allowed(td): + return td.__required_keys__ | td.__optional_keys__ + + +class TestNoUndeclaredKeys: + def test_text_part_keys(self): + d = llm.parts.TextPart( + text="hi", + provider_metadata={"k": "v"}, + ).to_dict() + assert set(d.keys()) <= _allowed(TextPartDict) + + def test_reasoning_part_keys(self): + d = llm.parts.ReasoningPart( + text="t", + redacted=True, + provider_metadata={"k": "v"}, + ).to_dict() + assert set(d.keys()) <= _allowed(ReasoningPartDict) + + def test_tool_call_part_keys(self): + d = llm.parts.ToolCallPart( + name="t", + arguments={"q": "x"}, + tool_call_id="c1", + server_executed=True, + provider_metadata={"k": "v"}, + ).to_dict() + assert set(d.keys()) <= _allowed(ToolCallPartDict) + + def test_tool_result_part_keys(self): + d = llm.parts.ToolResultPart( + name="t", + output="r", + tool_call_id="c1", + server_executed=True, + exception="boom", + attachments=[llm.Attachment(type="image/png", url="http://x/y.png")], + provider_metadata={"k": "v"}, + ).to_dict() + assert set(d.keys()) <= _allowed(ToolResultPartDict) + + def test_attachment_part_keys(self): + d = llm.parts.AttachmentPart( + attachment=llm.Attachment(type="image/png", url="http://x/y.png"), + provider_metadata={"k": "v"}, + ).to_dict() + assert set(d.keys()) <= _allowed(AttachmentPartDict) + + def test_message_keys(self): + d = llm.Message( + role="assistant", + parts=[llm.parts.TextPart(text="hi")], + provider_metadata={"k": "v"}, + ).to_dict() + assert set(d.keys()) <= _allowed(MessageDict) + + def test_response_keys(self, mock_model): + mock_model.enqueue(["answer"]) + r = mock_model.prompt("q", max_tokens=10) + r.text() + d = r.to_dict() + assert set(d.keys()) <= _allowed(ResponseDict) + # And the nested prompt sub-dict must conform too. + from llm.serialization import PromptDict + + assert set(d["prompt"].keys()) <= _allowed(PromptDict) From 926394aecd6c5631e5093063f30d1fa38715a92a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Apr 2026 17:46:07 -0700 Subject: [PATCH 067/258] Tweaked some overly-promotional language --- docs/plugins/advanced-model-plugins.md | 2 -- docs/python-api.md | 4 ++-- llm/default_plugins/openai_models.py | 32 ++++++++------------------ llm/models.py | 3 +-- tests/test_async_parity.py | 2 +- tests/test_parts.py | 2 +- 6 files changed, 14 insertions(+), 31 deletions(-) diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index 8d0027d96..6b562af8d 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -297,8 +297,6 @@ When you leave `part_index` as `None` (the default), the framework groups events - **Tool calls group by `tool_call_id`.** A `tool_call_name` and any number of `tool_call_args` events sharing a `tool_call_id` combine into one `ToolCallPart` — even if they're interleaved with other events (parallel tool calls). - **`tool_result` is always its own Part**, paired to the originating call by `tool_call_id`. -This handles every common shape without a plugin-side allocator: - | Stream | Resulting Parts | |-------------------------------------------|----------------------------------------------------------| | `text` × N | one `TextPart` | diff --git a/docs/python-api.md b/docs/python-api.md index 747d32789..5592135b9 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -606,7 +606,7 @@ On async models `messages()` is awaitable: `await response.messages()`. #### Persisting a conversation -A `Response` can round-trip through a plain Python dictionary via `response.to_dict()` and `llm.Response.from_dict(...)`. The dict captures the model id, the input messages that were sent, the assistant output, and any options — everything needed to continue the conversation later. +A `Response` can round-trip through a plain Python dictionary via `response.to_dict()` and `llm.Response.from_dict(...)`. The dict captures the model id, the input messages that were sent, the assistant output, and any options. The re-inflated object can be used to continue the conversation. Use `response.reply(...)` to continue from a rehydrated response: @@ -627,7 +627,7 @@ followup = rebuilt.reply("Add 3 to that") print(followup.text()) ``` -`AttachmentPart` bytes are base64-encoded in the dict form, so multi-modal conversations round-trip faithfully via JSON too. +`AttachmentPart` bytes are base64-encoded in the dict form, so multi-modal conversations round-trip via JSON too. Individual `Message` and `Part` objects also support `to_dict()` / `from_dict()` if you need to manipulate turns directly — for example, to edit, filter, or splice messages before passing them back via `model.prompt(messages=[...])`. diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 8e45c1145..f229635f1 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -832,13 +832,7 @@ def _append_llm_message(self, out, message, current_system, image_detail=None): return current_system def build_messages(self, prompt, conversation, image_detail=None): - """Translate prompt.messages into OpenAI's wire format. - - Under the Phase 7 invariant, ``prompt.messages`` is the full - chain for this turn — Conversation.prompt and response.reply - pre-bake the history into it. The ``conversation`` parameter - is unused and retained only for the plugin API contract. - """ + """Translate prompt.messages into OpenAI's wire format.""" messages: List[Dict[str, Any]] = [] if image_detail is not None: image_detail = image_detail.value @@ -1024,16 +1018,11 @@ def execute( type="text", chunk=completion.choices[0].message.content, ) - # Read reasoning_tokens from usage BEFORE set_usage runs — - # set_usage pops top-level keys and passes the rest through - # simplify_usage_dict, which strips zero-valued entries. - if usage: - reasoning_tokens = (usage.get("completion_tokens_details") or {}).get( - "reasoning_tokens", 0 - ) - if reasoning_tokens: - yield StreamEvent(type="reasoning", chunk="", redacted=True) self.set_usage(response, usage) + if usage and (usage.get("completion_tokens_details") or {}).get( + "reasoning_tokens" + ): + yield StreamEvent(type="reasoning", chunk="", redacted=True) response._prompt_json = redact_data({"messages": messages}) @@ -1145,14 +1134,11 @@ async def execute( type="text", chunk=completion.choices[0].message.content, ) - # See sync Chat.execute: read reasoning before set_usage mutates. - if usage: - reasoning_tokens = (usage.get("completion_tokens_details") or {}).get( - "reasoning_tokens", 0 - ) - if reasoning_tokens: - yield StreamEvent(type="reasoning", chunk="", redacted=True) self.set_usage(response, usage) + if usage and (usage.get("completion_tokens_details") or {}).get( + "reasoning_tokens" + ): + yield StreamEvent(type="reasoning", chunk="", redacted=True) response._prompt_json = redact_data({"messages": messages}) diff --git a/llm/models.py b/llm/models.py index 7829a1855..13ca5c3db 100644 --- a/llm/models.py +++ b/llm/models.py @@ -1607,8 +1607,7 @@ def reply( Builds the next turn's chain as ``self.prompt.messages + self.messages + [tool_message] + [user(prompt)] + messages`` and calls - ``self.model.prompt(messages=chain, ...)``. No Conversation - object required — the Response carries everything needed. + ``self.model.prompt(messages=chain, ...)``. If this response made tool calls and ``tool_results=`` is not passed, ``reply()`` runs ``self.execute_tool_calls()`` diff --git a/tests/test_async_parity.py b/tests/test_async_parity.py index 0ccaa8394..b20fb6ecc 100644 --- a/tests/test_async_parity.py +++ b/tests/test_async_parity.py @@ -63,7 +63,7 @@ async def test_async_from_dict_rehydrates(): @pytest.mark.asyncio async def test_async_from_dict_then_reply_continues(): - """The whole point: persist an async response across process + """Persist an async response across process boundary (via JSON), rehydrate, continue with reply().""" model = llm.get_async_model("echo") r1 = model.prompt("q1") diff --git a/tests/test_parts.py b/tests/test_parts.py index e23f92d04..fbe67ecc1 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -1250,7 +1250,7 @@ async def test_async_reply(self, async_mock_model): ] def test_reply_with_tool_results_appends_tool_message(self, mock_model): - # The natural idiom: model.prompt(...) makes tool calls, the + # model.prompt(...) makes tool calls, the # caller runs them, then reply(tool_results=...) sends the # results back in one call. The chain should grow by a # role="tool" message containing ToolResultParts. From 35c35dac43c232e1e3fca274812bd65818d3ed5d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Apr 2026 11:52:38 -0700 Subject: [PATCH 068/258] Release 0.32a0 Closes #1314, #506, #1278, #894, #813, #468, #346, #716, #770, #867, #938, #1033, #937 Refs #1067, #1080 --- docs/changelog.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 834d50b39..715593809 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -5,6 +5,8 @@ This alpha introduces a major backwards-compatible refactor. Models can now be prompted with a list of messages, OpenAI Chat Completions style, and the response can now be iterated over as a sequence of mixed types of content, for example reasoning tokens mixed with text tokens mixed with tool calls. +For more background on this release take a look at [the annotated release notes](https://simonwillison.net/2026/Apr/29/llm/) on my blog. + Prompt inputs and response outputs are now expressed as a list of `Message` objects, each containing typed `Part` objects (text, reasoning, tool calls, tool results, attachments). The `llm` CLI tool can now display reasoning tokens while executing a prompt. diff --git a/pyproject.toml b/pyproject.toml index dc73954d7..b008860b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm" -version = "0.31" +version = "0.32a0" description = "CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine." readme = { file = "README.md", content-type = "text/markdown" } authors = [ From cce6ed956a703cac21453d83bffb556ae45ae9e1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Apr 2026 12:02:36 -0700 Subject: [PATCH 069/258] Ran cog --- docs/fragments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/fragments.md b/docs/fragments.md index a7e908a9a..ea1bbbb50 100644 --- a/docs/fragments.md +++ b/docs/fragments.md @@ -30,7 +30,7 @@ from importlib.metadata import version llm_version = version("llm") cog.out(f'The URL will be fetched with the user-agent `llm/{llm_version} (https://llm.datasette.io/)`.') ]]]--> -The URL will be fetched with the user-agent `llm/0.31 (https://llm.datasette.io/)`. +The URL will be fetched with the user-agent `llm/0.32a0 (https://llm.datasette.io/)`. The `-f` option can be used multiple times to combine together multiple fragments. From 4d92df12a6a97d06776cc01780ed1287f7424bbb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Apr 2026 16:37:43 -0700 Subject: [PATCH 070/258] Rebuild prompt.messages chain when loading logged conversations Each row stores only its current-turn inputs, so a loaded tool-result response began with an orphan tool_result. `llm -c` then sent a request with an unexpected tool_use_id. Stitch each response's messages onto the previous response's chain plus its assistant output during load. Closes #1426 Refs https://github.com/simonw/llm-anthropic/issues/68 Co-Authored-By: Claude Opus 4.7 (1M context) --- llm/cli.py | 16 ++++++++++++-- tests/test_parts.py | 52 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index 386b35bdb..ead5d0a43 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -1311,9 +1311,21 @@ def load_conversation( response_class = AsyncResponse if async_ else Response conversation = conversation_class.from_row(row) for response in db["responses"].rows_where( - "conversation_id = ?", [conversation_id] + "conversation_id = ?", [conversation_id], order_by="id" ): - conversation.responses.append(response_class.from_row(db, response)) + response_obj = response_class.from_row(db, response) + if conversation.responses: + previous_response = conversation.responses[-1] + # SQLite rows store each response's legacy current-turn inputs + # (prompt text, attachments, tool_results), not the full + # prompt.messages chain. Rebuild that chain here so follow-up + # prompts via `llm -c` satisfy the Prompt.messages invariant. + response_obj.prompt._explicit_messages = ( + list(previous_response.prompt.messages) + + list(previous_response._messages_now()) + + list(response_obj.prompt.messages) + ) + conversation.responses.append(response_obj) return conversation diff --git a/tests/test_parts.py b/tests/test_parts.py index fbe67ecc1..44687d75d 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -1172,6 +1172,58 @@ def test_llm_dash_c_chain_preserves_prior_assistant_turn( llm.user("q2"), ] + def test_llm_dash_c_after_logged_tool_chain_preserves_full_chain( + self, mock_model, tmp_path + ): + """A loaded tool-result response must carry the preceding + assistant tool_use. Otherwise Anthropic sees an orphan + tool_result at the start of the continued request.""" + import sqlite_utils + from llm.cli import load_conversation + from llm.migrations import migrate + + class ToolChainMock(type(mock_model)): + def __init__(self): + super().__init__() + self.calls = 0 + + def execute(self, prompt, stream, response, conversation): + self.calls += 1 + if self.calls == 1: + response.add_tool_call( + llm.ToolCall(name="tick", arguments={}, tool_call_id="c1") + ) + if False: + yield "" + else: + yield "final answer" + + def tick() -> str: + return "tock" + + m = ToolChainMock() + chain_response = m.chain("q1", tools=[tick]) + chain_response.text() + + db_path = tmp_path / "logs.db" + db = sqlite_utils.Database(str(db_path)) + migrate(db) + chain_response.log_to_db(db) + + conv = load_conversation(None, database=str(db_path)) + r3 = conv.prompt("q2") + + assert [m.role for m in r3.prompt.messages] == [ + "user", + "assistant", + "tool", + "assistant", + "user", + ] + assert isinstance(r3.prompt.messages[1].parts[0], llm.parts.ToolCallPart) + assert isinstance(r3.prompt.messages[2].parts[0], llm.parts.ToolResultPart) + assert r3.prompt.messages[2].parts[0].tool_call_id == "c1" + class TestResponseReply: def test_reply_builds_next_turn_from_this_response(self, mock_model): From 9a5c24e20cbcc8ba24dbc112e2af32e6be7a1b8f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Apr 2026 16:52:09 -0700 Subject: [PATCH 071/258] Release 0.32a1 Refs #1426 Refs https://github.com/simonw/llm-anthropic/issues/68 --- docs/changelog.md | 5 +++++ docs/fragments.md | 2 +- pyproject.toml | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 715593809..b82490464 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,10 @@ # Changelog +(v0_32_a1)= +## 0.32a1 (2026-04-29) + +- Fixed a bug in 0.32a0 where tool-calling conversations were not correctly reinflated from SQLite. [#1426](https://github.com/simonw/llm/issues/1426) + (v0_32_a0)= ## 0.32a0 (2026-04-28) diff --git a/docs/fragments.md b/docs/fragments.md index ea1bbbb50..935bbab32 100644 --- a/docs/fragments.md +++ b/docs/fragments.md @@ -30,7 +30,7 @@ from importlib.metadata import version llm_version = version("llm") cog.out(f'The URL will be fetched with the user-agent `llm/{llm_version} (https://llm.datasette.io/)`.') ]]]--> -The URL will be fetched with the user-agent `llm/0.32a0 (https://llm.datasette.io/)`. +The URL will be fetched with the user-agent `llm/0.32a1 (https://llm.datasette.io/)`. The `-f` option can be used multiple times to combine together multiple fragments. diff --git a/pyproject.toml b/pyproject.toml index b008860b7..9ca5d3c22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm" -version = "0.32a0" +version = "0.32a1" description = "CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine." readme = { file = "README.md", content-type = "text/markdown" } authors = [ From 3d0321fbb4c13f0d23d66f042ebbeea6ca7ccfad Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 4 May 2026 20:57:01 -0700 Subject: [PATCH 072/258] Add options= dict parameter to .prompt() and .reply() (#1432) Accepts model options as an explicit dict alongside the existing **kwargs form. The kwargs form continues to work unchanged for backwards compatibility but is no longer documented. Mixing the two forms with overlapping keys raises TypeError. Applies to Model.prompt, Conversation.prompt, Response.reply and their async equivalents. .chain() already used this pattern. Co-authored-by: Claude --- docs/python-api.md | 4 +- llm/models.py | 46 +++++++++++---- tests/test_options_parameter.py | 99 +++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 14 deletions(-) create mode 100644 tests/test_options_parameter.py diff --git a/docs/python-api.md b/docs/python-api.md index 5592135b9..bbf0910b8 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -411,11 +411,11 @@ Some model plugins may include features that take advantage of fragments, for ex ### Model options -For models that support options (view those with `llm models --options`) you can pass options as keyword arguments to the `.prompt()` method: +For models that support options (view those with `llm models --options`) pass them as a dictionary to the `options=` argument of the `.prompt()` method: ```python model = llm.get_model() -print(model.prompt("Names for otters", temperature=0.2)) +print(model.prompt("Names for otters", options={"temperature": 0.2})) ``` (python-api-models-api-keys)= diff --git a/llm/models.py b/llm/models.py index 13ca5c3db..20f044338 100644 --- a/llm/models.py +++ b/llm/models.py @@ -484,6 +484,18 @@ def _wrap_tools(tools: List[ToolDef]) -> List[Tool]: return wrapped_tools +def _merge_options(options: Optional[dict], kwargs: dict) -> dict: + if not options: + return kwargs + overlap = set(options) & set(kwargs) + if overlap: + raise TypeError( + "Got values for these options both in options= and as keyword " + "arguments: {}".format(sorted(overlap)) + ) + return {**options, **kwargs} + + @dataclass class _BaseConversation: model: "_BaseModel" @@ -583,8 +595,10 @@ def prompt( messages: Optional[List[Any]] = None, stream: bool = True, key: Optional[str] = None, - **options, + options: Optional[dict] = None, + **kwargs, ) -> "Response": + merged = _merge_options(options, kwargs) # Build the authoritative chain so response.prompt.messages # equals exactly what the model sees for this turn. chain = self._build_full_chain( @@ -605,7 +619,7 @@ def prompt( tool_results=tool_results, system_fragments=system_fragments, messages=chain, - options=self.model.Options(**options), + options=self.model.Options(**merged), ), self.model, stream, @@ -750,8 +764,10 @@ def prompt( messages: Optional[List[Any]] = None, stream: bool = True, key: Optional[str] = None, - **options, + options: Optional[dict] = None, + **kwargs, ) -> "AsyncResponse": + merged = _merge_options(options, kwargs) chain = self._build_full_chain( prompt=prompt, attachments=attachments, @@ -770,7 +786,7 @@ def prompt( tool_results=tool_results, system_fragments=system_fragments, messages=chain, - options=self.model.Options(**options), + options=self.model.Options(**merged), ), self.model, stream, @@ -1600,6 +1616,7 @@ def reply( *, messages: Optional[List[Any]] = None, tool_results: Optional[List[ToolResult]] = None, + options: Optional[dict] = None, **kwargs, ) -> "Response": """Continue the conversation from this response. @@ -1643,7 +1660,7 @@ def reply( chain.append(Message(role="user", parts=[TextPart(text=prompt)])) if messages: chain.extend(messages) - return self.model.prompt(messages=chain, **kwargs) + return self.model.prompt(messages=chain, options=options, **kwargs) def to_dict(self) -> ResponseDict: """Serialize this response for JSON persistence. @@ -1937,6 +1954,7 @@ async def reply( *, messages: Optional[List[Any]] = None, tool_results: Optional[List[ToolResult]] = None, + options: Optional[dict] = None, **kwargs, ) -> "AsyncResponse": """Async counterpart of Response.reply(). Requires this response @@ -1975,7 +1993,7 @@ async def reply( chain.append(Message(role="user", parts=[TextPart(text=prompt)])) if messages: chain.extend(messages) - return self.model.prompt(messages=chain, **kwargs) + return self.model.prompt(messages=chain, options=options, **kwargs) def to_dict(self) -> ResponseDict: """Async counterpart of Response.to_dict(). Requires awaiting.""" @@ -2739,9 +2757,11 @@ def prompt( schema: Optional[Union[dict, type[BaseModel]]] = None, tools: Optional[List[ToolDef]] = None, tool_results: Optional[List[ToolResult]] = None, - **options, + options: Optional[dict] = None, + **kwargs, ) -> Response: - key_value = options.pop("key", None) + key_value = kwargs.pop("key", None) + merged = _merge_options(options, kwargs) self._validate_attachments(attachments) return Response( Prompt( @@ -2755,7 +2775,7 @@ def prompt( system_fragments=system_fragments, messages=messages, model=self, - options=self.Options(**options), + options=self.Options(**merged), ), self, stream, @@ -2852,9 +2872,11 @@ def prompt( system_fragments: Optional[List[Union[str, Fragment]]] = None, messages: Optional[List[Any]] = None, stream: bool = True, - **options, + options: Optional[dict] = None, + **kwargs, ) -> AsyncResponse: - key_value = options.pop("key", None) + key_value = kwargs.pop("key", None) + merged = _merge_options(options, kwargs) self._validate_attachments(attachments) return AsyncResponse( Prompt( @@ -2868,7 +2890,7 @@ def prompt( system_fragments=system_fragments, messages=messages, model=self, - options=self.Options(**options), + options=self.Options(**merged), ), self, stream, diff --git a/tests/test_options_parameter.py b/tests/test_options_parameter.py new file mode 100644 index 000000000..df4c09a3a --- /dev/null +++ b/tests/test_options_parameter.py @@ -0,0 +1,99 @@ +"""Tests for the `options=` parameter on `.prompt()` and `.reply()`. + +The `options={...}` dict form is the documented API; the `**kwargs` form +continues to work undocumented for backwards compatibility. +""" + +import pytest + + +def test_prompt_with_options_dict(mock_model): + mock_model.enqueue(["ok"]) + r = mock_model.prompt("q", options={"max_tokens": 42}) + r.text() + assert r.prompt.options.max_tokens == 42 + assert r.to_dict()["prompt"].get("options") == {"max_tokens": 42} + + +def test_prompt_kwargs_still_work(mock_model): + mock_model.enqueue(["ok"]) + r = mock_model.prompt("q", max_tokens=42) + r.text() + assert r.prompt.options.max_tokens == 42 + + +def test_prompt_options_and_kwargs_merge(mock_model): + # Non-overlapping keys merge cleanly — options= and kwargs both contribute + mock_model.Options.model_rebuild() + mock_model.enqueue(["ok"]) + # Only max_tokens exists on MockModel.Options — use it via options=. + # Pass an empty options dict alongside a kwarg to confirm both paths coexist. + r = mock_model.prompt("q", options={}, max_tokens=7) + r.text() + assert r.prompt.options.max_tokens == 7 + + +def test_prompt_options_and_kwargs_conflict_raises(mock_model): + mock_model.enqueue(["ok"]) + with pytest.raises(TypeError, match="both in options="): + mock_model.prompt("q", options={"max_tokens": 1}, max_tokens=2) + + +def test_conversation_prompt_with_options_dict(mock_model): + mock_model.enqueue(["ok"]) + convo = mock_model.conversation() + r = convo.prompt("q", options={"max_tokens": 99}) + r.text() + assert r.prompt.options.max_tokens == 99 + + +def test_response_reply_with_options_dict(mock_model): + mock_model.enqueue(["first"]) + mock_model.enqueue(["second"]) + r1 = mock_model.prompt("q1", options={"max_tokens": 5}) + r1.text() + r2 = r1.reply("q2", options={"max_tokens": 17}) + r2.text() + assert r2.prompt.options.max_tokens == 17 + + +def test_response_reply_kwargs_still_work(mock_model): + mock_model.enqueue(["first"]) + mock_model.enqueue(["second"]) + r1 = mock_model.prompt("q1", max_tokens=5) + r1.text() + r2 = r1.reply("q2", max_tokens=17) + r2.text() + assert r2.prompt.options.max_tokens == 17 + + +@pytest.mark.asyncio +async def test_async_prompt_with_options_dict(async_mock_model): + # AsyncMockModel inherits the empty base Options (extra="forbid"), + # so pass an empty options dict — this verifies the parameter is + # accepted and the empty-dict path works. + async_mock_model.enqueue(["ok"]) + r = await async_mock_model.prompt("q", options={}).text() + assert r == "ok" + + +@pytest.mark.asyncio +async def test_async_prompt_options_and_kwargs_conflict_raises(async_mock_model): + import llm + + # Build an async model with a real Option field so we can collide them. + class AsyncModelWithOption(llm.AsyncModel): + model_id = "async-with-option" + + class Options(llm.Options): + from typing import Optional as _Opt + from pydantic import Field as _Field + + max_tokens: _Opt[int] = _Field(default=None) + + async def execute(self, prompt, stream, response, conversation): + yield "ok" + + m = AsyncModelWithOption() + with pytest.raises(TypeError, match="both in options="): + await m.prompt("q", options={"max_tokens": 1}, max_tokens=2).text() From 3c747c8b9f96f8ff9a7c611a579726cb3ef2ff71 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 01:57:09 +0000 Subject: [PATCH 073/258] Route gpt-5.5 through the /v1/responses endpoint Adds Responses and AsyncResponses classes that drive the OpenAI /v1/responses endpoint. The existing Chat / AsyncChat classes are unchanged because other plugins import them. gpt-5.5 (and gpt-5.5-2026-04-23) is now registered against Responses by default. Pass `-o chat_completions 1` to fall back to the older /v1/chat/completions code path. This is feature parity with the Chat path (text, tools, streaming, schema, reasoning_effort, verbosity, attachments, system prompts). Interleaved reasoning across tool round-trips is not exercised yet - encrypted reasoning items are accepted on the input side, but the plugin doesn't yet stash them on outgoing ReasoningParts. --- llm/default_plugins/openai_models.py | 675 +++++++++++++++++- .../test_responses_basic_non_streaming.yaml | 105 +++ .../test_responses_basic_streaming.yaml | 150 ++++ .../test_responses_tool_use.yaml | 216 ++++++ .../test_responses_tool_use_streaming.yaml | 393 ++++++++++ tests/test_openai_responses.py | 219 ++++++ 6 files changed, 1754 insertions(+), 4 deletions(-) create mode 100644 tests/cassettes/test_openai_responses/test_responses_basic_non_streaming.yaml create mode 100644 tests/cassettes/test_openai_responses/test_responses_basic_streaming.yaml create mode 100644 tests/cassettes/test_openai_responses/test_responses_tool_use.yaml create mode 100644 tests/cassettes/test_openai_responses/test_responses_tool_use_streaming.yaml create mode 100644 tests/test_openai_responses.py diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index f229635f1..3d62a8f95 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -279,13 +279,14 @@ def register_models(register): supports_tools=True, ), ) - # GPT-5.5 + # GPT-5.5 — routes through the Responses API by default; pass + # ``-o chat_completions 1`` to fall back to /v1/chat/completions. for model_id in ( "gpt-5.5", "gpt-5.5-2026-04-23", ): register( - Chat( + Responses( model_id, vision=True, reasoning=True, @@ -294,7 +295,7 @@ def register_models(register): supports_schema=True, supports_tools=True, ), - AsyncChat( + AsyncResponses( model_id, vision=True, reasoning=True, @@ -599,7 +600,11 @@ def enum_values_sentence(enum_class): def build_options_class( - *, reasoning=False, verbosity=False, image_detail_original=False + *, + reasoning=False, + verbosity=False, + image_detail_original=False, + chat_completions=False, ): fields = { "json_object": ( @@ -610,6 +615,19 @@ def build_options_class( ), ) } + if chat_completions: + fields["chat_completions"] = ( + Optional[bool], + Field( + description=( + "Force the use of the older /v1/chat/completions endpoint " + "instead of /v1/responses. Most callers should leave this " + "off; set to true to fall back to the Chat Completions code " + "path for compatibility." + ), + default=None, + ), + ) image_detail_enum = ( ImageDetailWithOriginalEnum if image_detail_original else ImageDetailEnum ) @@ -882,6 +900,7 @@ def build_kwargs(self, prompt, stream): kwargs = dict(not_nulls(prompt.options)) json_object = kwargs.pop("json_object", None) kwargs.pop("image_detail", None) + kwargs.pop("chat_completions", None) if "max_tokens" not in kwargs and self.default_max_tokens is not None: kwargs["max_tokens"] = self.default_max_tokens if json_object: @@ -1142,6 +1161,654 @@ async def execute( response._prompt_json = redact_data({"messages": messages}) +def _responses_attachment(attachment, image_detail=None): + """Translate an llm Attachment into a Responses-API content part.""" + url = attachment.url + base64_content = "" + if not url or attachment.resolve_type().startswith("audio/"): + base64_content = attachment.base64_content() + url = f"data:{attachment.resolve_type()};base64,{base64_content}" + if attachment.resolve_type() == "application/pdf": + if not base64_content: + base64_content = attachment.base64_content() + return { + "type": "input_file", + "filename": f"{attachment.id()}.pdf", + "file_data": f"data:application/pdf;base64,{base64_content}", + } + if attachment.resolve_type().startswith("image/"): + item = {"type": "input_image", "image_url": url} + if image_detail: + item["detail"] = image_detail + return item + # Audio is not yet supported on the Responses input shape we use; fall + # back to image_url for unknown types so we don't silently drop content. + return {"type": "input_image", "image_url": url} + + +class _SharedResponses(_Shared): + """Mixin that translates llm.Prompt into Responses API parameters.""" + + def __str__(self) -> str: + return "OpenAI Responses: {}".format(self.model_id) + + def _delegate_chat_kwargs(self): + """Return constructor kwargs that mirror this Responses model so we + can build a sibling Chat / AsyncChat instance for the + ``-o chat_completions 1`` opt-out path.""" + return dict( + model_id=self.model_id, + key=self.key, + model_name=self.model_name, + api_base=self.api_base, + api_type=self.api_type, + api_version=self.api_version, + api_engine=self.api_engine, + headers=self.headers, + can_stream=self.can_stream, + vision=self.vision, + reasoning=self._reasoning, + verbosity=self._verbosity, + image_detail_original=self._image_detail_original, + supports_schema=self.supports_schema, + supports_tools=self.supports_tools, + allows_system_prompt=self.allows_system_prompt, + ) + + def _build_responses_input(self, prompt, image_detail=None): + """Translate prompt.messages into a (input_items, instructions) tuple + for the Responses API. + + The most recent system Message is hoisted into ``instructions``; + earlier system messages are dropped (mirroring the way the Chat + path collapses repeated identical system prompts). + """ + from llm.parts import ( + AttachmentPart, + ReasoningPart, + TextPart, + ToolCallPart, + ToolResultPart, + ) + + items: List[Dict[str, Any]] = [] + instructions: Optional[str] = None + + for msg in prompt.messages: + if msg.role == "system": + text = "".join( + p.text for p in msg.parts if isinstance(p, TextPart) + ) + if text: + instructions = text + continue + + text_bits: List[str] = [] + attachment_items: List[Dict[str, Any]] = [] + tool_call_items: List[Dict[str, Any]] = [] + tool_result_items: List[Dict[str, Any]] = [] + reasoning_items: List[Dict[str, Any]] = [] + + for part in msg.parts: + if isinstance(part, TextPart): + text_bits.append(part.text) + elif isinstance(part, AttachmentPart) and part.attachment: + attachment_items.append( + _responses_attachment( + part.attachment, image_detail=image_detail + ) + ) + elif isinstance(part, ToolCallPart): + tool_call_items.append( + { + "type": "function_call", + "call_id": part.tool_call_id, + "name": part.name, + "arguments": json.dumps(part.arguments), + } + ) + elif isinstance(part, ToolResultPart): + tool_result_items.append( + { + "type": "function_call_output", + "call_id": part.tool_call_id, + "output": part.output, + } + ) + elif isinstance(part, ReasoningPart): + pm = (part.provider_metadata or {}).get("openai") or {} + enc = pm.get("encrypted_content") + rid = pm.get("id") + if enc or rid: + # Round-trip a previous reasoning item so the model + # can pick up where it left off mid-tool-call. + item: Dict[str, Any] = {"type": "reasoning"} + if rid: + item["id"] = rid + if enc: + item["encrypted_content"] = enc + if pm.get("summary"): + item["summary"] = pm["summary"] + else: + item["summary"] = [] + reasoning_items.append(item) + + # Reasoning items must precede the assistant message / function + # call they belonged to. + items.extend(reasoning_items) + + if msg.role == "tool": + items.extend(tool_result_items) + continue + + if msg.role == "user": + if attachment_items: + content: List[Dict[str, Any]] = [] + if text_bits: + content.append( + {"type": "input_text", "text": "".join(text_bits)} + ) + content.extend(attachment_items) + items.append({"role": "user", "content": content}) + elif text_bits: + items.append({"role": "user", "content": "".join(text_bits)}) + elif msg.role == "assistant": + if text_bits: + items.append( + { + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "".join(text_bits), + } + ], + } + ) + items.extend(tool_call_items) + + return items, instructions + + def _build_responses_kwargs(self, prompt, stream): + """Build the keyword arguments for client.responses.create().""" + opts = dict(not_nulls(prompt.options)) + # Strip options that are either internal to llm or not accepted by + # the Responses API. + opts.pop("json_object", None) + opts.pop("chat_completions", None) + opts.pop("image_detail", None) + max_tokens = opts.pop("max_tokens", None) + reasoning_effort = opts.pop("reasoning_effort", None) + verbosity = opts.pop("verbosity", None) + temperature = opts.pop("temperature", None) + top_p = opts.pop("top_p", None) + seed = opts.pop("seed", None) + + kwargs: Dict[str, Any] = {} + if max_tokens is None and self.default_max_tokens is not None: + max_tokens = self.default_max_tokens + if max_tokens is not None: + kwargs["max_output_tokens"] = max_tokens + if temperature is not None: + kwargs["temperature"] = temperature + if top_p is not None: + kwargs["top_p"] = top_p + if seed is not None: + kwargs["seed"] = seed + if reasoning_effort: + kwargs["reasoning"] = {"effort": reasoning_effort} + + text: Dict[str, Any] = {} + if verbosity: + text["verbosity"] = verbosity + if prompt.options.json_object: + text["format"] = {"type": "json_object"} + if prompt.schema: + # ``strict: False`` mirrors the looser behaviour of the + # /v1/chat/completions json_schema response_format - required + # because the Responses API otherwise insists on + # ``additionalProperties: false`` everywhere. + text["format"] = { + "type": "json_schema", + "name": "output", + "schema": prompt.schema, + "strict": False, + } + if text: + kwargs["text"] = text + + if prompt.tools: + kwargs["tools"] = [ + { + "type": "function", + "name": tool.name, + "description": tool.description or None, + "parameters": tool.input_schema, + } + for tool in prompt.tools + ] + + # Pass anything we did not consume through verbatim - this lets + # extras like ``parallel_tool_calls`` flow into the API. + kwargs.update(opts) + return kwargs + + def _set_usage_responses(self, response, usage): + if not usage: + return + input_tokens = usage.get("input_tokens", 0) or 0 + output_tokens = usage.get("output_tokens", 0) or 0 + details = {} + for key in ("input_tokens_details", "output_tokens_details"): + value = usage.get(key) + if value: + details[key] = value + response.set_usage( + input=input_tokens, output=output_tokens, details=details or None + ) + + +class Responses(_SharedResponses, KeyModel): + needs_key = "openai" + key_env_var = "OPENAI_API_KEY" + default_max_tokens = None + + def __init__( + self, + model_id, + key=None, + model_name=None, + api_base=None, + api_type=None, + api_version=None, + api_engine=None, + headers=None, + can_stream=True, + vision=False, + audio=False, + reasoning=False, + verbosity=False, + image_detail_original=False, + supports_schema=False, + supports_tools=False, + allows_system_prompt=True, + ): + super().__init__( + model_id, + key=key, + model_name=model_name, + api_base=api_base, + api_type=api_type, + api_version=api_version, + api_engine=api_engine, + headers=headers, + can_stream=can_stream, + vision=vision, + audio=audio, + reasoning=reasoning, + verbosity=verbosity, + image_detail_original=image_detail_original, + supports_schema=supports_schema, + supports_tools=supports_tools, + allows_system_prompt=allows_system_prompt, + ) + self._reasoning = reasoning + self._verbosity = verbosity + self._image_detail_original = image_detail_original + # Override the Options class so that ``-o chat_completions 1`` is + # always available on Responses-routed models. + self.Options = build_options_class( + reasoning=reasoning, + verbosity=verbosity, + image_detail_original=image_detail_original, + chat_completions=True, + ) + + def execute( + self, + prompt: Prompt, + stream: bool, + response: Response, + conversation: Optional[Conversation] = None, + key: Optional[str] = None, + ) -> Iterator[Union[str, StreamEvent]]: + if getattr(prompt.options, "chat_completions", None): + chat = Chat(**self._delegate_chat_kwargs()) + yield from chat.execute(prompt, stream, response, conversation, key) + return + + if prompt.system and not self.allows_system_prompt: + raise NotImplementedError("Model does not support system prompts") + + image_detail = getattr(prompt.options, "image_detail", None) + if image_detail is not None: + image_detail = image_detail.value + input_items, instructions = self._build_responses_input( + prompt, image_detail=image_detail + ) + kwargs = self._build_responses_kwargs(prompt, stream) + if instructions is not None: + kwargs["instructions"] = instructions + kwargs["store"] = False + kwargs["include"] = ["reasoning.encrypted_content"] + + client = self.get_client(key) + usage = None + had_reasoning = False + if stream: + stream_obj = client.responses.create( + model=self.model_name or self.model_id, + input=input_items, + stream=True, + **kwargs, + ) + tool_call_meta: Dict[str, Dict[str, str]] = {} + final_response_dict: Optional[Dict[str, Any]] = None + for event in stream_obj: + etype = getattr(event, "type", None) + if etype == "response.output_item.added": + item = event.item + if item.type == "function_call": + tool_call_meta[item.id] = { + "id": item.id, + "call_id": item.call_id, + "name": item.name, + } + yield StreamEvent( + type="tool_call_name", + chunk=item.name or "", + tool_call_id=item.call_id, + ) + elif item.type == "reasoning": + had_reasoning = True + elif etype == "response.output_text.delta": + yield StreamEvent(type="text", chunk=event.delta or "") + elif etype == "response.function_call_arguments.delta": + item_id = getattr(event, "item_id", None) + meta = tool_call_meta.get(item_id) if item_id else None + call_id = meta["call_id"] if meta else None + yield StreamEvent( + type="tool_call_args", + chunk=event.delta or "", + tool_call_id=call_id, + ) + elif etype == "response.output_item.done": + item = event.item + if item.type == "function_call": + try: + args = ( + json.loads(item.arguments) if item.arguments else {} + ) + except json.JSONDecodeError: + args = {"_raw": item.arguments} + response.add_tool_call( + llm.ToolCall( + tool_call_id=item.call_id, + name=item.name, + arguments=args, + ) + ) + elif etype == "response.completed": + final_response_dict = event.response.model_dump() + if final_response_dict.get("usage"): + usage = final_response_dict["usage"] + if final_response_dict is not None: + response.response_json = remove_dict_none_values( + final_response_dict + ) + else: + completion = client.responses.create( + model=self.model_name or self.model_id, + input=input_items, + stream=False, + **kwargs, + ) + dumped = completion.model_dump() + response.response_json = remove_dict_none_values(dumped) + usage = dumped.get("usage") + for item in completion.output: + if item.type == "reasoning": + had_reasoning = True + elif item.type == "function_call": + try: + args = json.loads(item.arguments) if item.arguments else {} + except json.JSONDecodeError: + args = {"_raw": item.arguments} + response.add_tool_call( + llm.ToolCall( + tool_call_id=item.call_id, + name=item.name, + arguments=args, + ) + ) + yield StreamEvent( + type="tool_call_name", + chunk=item.name or "", + tool_call_id=item.call_id, + ) + yield StreamEvent( + type="tool_call_args", + chunk=item.arguments or "", + tool_call_id=item.call_id, + ) + elif item.type == "message": + for content in item.content or []: + ctype = getattr(content, "type", None) + if ctype == "output_text" and content.text: + yield StreamEvent(type="text", chunk=content.text) + + self._set_usage_responses(response, usage) + if had_reasoning or ( + usage + and (usage.get("output_tokens_details") or {}).get("reasoning_tokens") + ): + yield StreamEvent(type="reasoning", chunk="", redacted=True) + response._prompt_json = redact_data( + {"input": input_items, "instructions": instructions} + ) + + +class AsyncResponses(_SharedResponses, AsyncKeyModel): + needs_key = "openai" + key_env_var = "OPENAI_API_KEY" + default_max_tokens = None + + def __init__( + self, + model_id, + key=None, + model_name=None, + api_base=None, + api_type=None, + api_version=None, + api_engine=None, + headers=None, + can_stream=True, + vision=False, + audio=False, + reasoning=False, + verbosity=False, + image_detail_original=False, + supports_schema=False, + supports_tools=False, + allows_system_prompt=True, + ): + super().__init__( + model_id, + key=key, + model_name=model_name, + api_base=api_base, + api_type=api_type, + api_version=api_version, + api_engine=api_engine, + headers=headers, + can_stream=can_stream, + vision=vision, + audio=audio, + reasoning=reasoning, + verbosity=verbosity, + image_detail_original=image_detail_original, + supports_schema=supports_schema, + supports_tools=supports_tools, + allows_system_prompt=allows_system_prompt, + ) + self._reasoning = reasoning + self._verbosity = verbosity + self._image_detail_original = image_detail_original + self.Options = build_options_class( + reasoning=reasoning, + verbosity=verbosity, + image_detail_original=image_detail_original, + chat_completions=True, + ) + + async def execute( + self, + prompt: Prompt, + stream: bool, + response: AsyncResponse, + conversation: Optional[AsyncConversation] = None, + key: Optional[str] = None, + ) -> AsyncGenerator[Union[str, StreamEvent], None]: + if getattr(prompt.options, "chat_completions", None): + chat = AsyncChat(**self._delegate_chat_kwargs()) + async for event in chat.execute( + prompt, stream, response, conversation, key + ): + yield event + return + + if prompt.system and not self.allows_system_prompt: + raise NotImplementedError("Model does not support system prompts") + + image_detail = getattr(prompt.options, "image_detail", None) + if image_detail is not None: + image_detail = image_detail.value + input_items, instructions = self._build_responses_input( + prompt, image_detail=image_detail + ) + kwargs = self._build_responses_kwargs(prompt, stream) + if instructions is not None: + kwargs["instructions"] = instructions + kwargs["store"] = False + kwargs["include"] = ["reasoning.encrypted_content"] + + client = self.get_client(key, async_=True) + usage = None + had_reasoning = False + if stream: + stream_obj = await client.responses.create( + model=self.model_name or self.model_id, + input=input_items, + stream=True, + **kwargs, + ) + tool_call_meta: Dict[str, Dict[str, str]] = {} + final_response_dict: Optional[Dict[str, Any]] = None + async for event in stream_obj: + etype = getattr(event, "type", None) + if etype == "response.output_item.added": + item = event.item + if item.type == "function_call": + tool_call_meta[item.id] = { + "id": item.id, + "call_id": item.call_id, + "name": item.name, + } + yield StreamEvent( + type="tool_call_name", + chunk=item.name or "", + tool_call_id=item.call_id, + ) + elif item.type == "reasoning": + had_reasoning = True + elif etype == "response.output_text.delta": + yield StreamEvent(type="text", chunk=event.delta or "") + elif etype == "response.function_call_arguments.delta": + item_id = getattr(event, "item_id", None) + meta = tool_call_meta.get(item_id) if item_id else None + call_id = meta["call_id"] if meta else None + yield StreamEvent( + type="tool_call_args", + chunk=event.delta or "", + tool_call_id=call_id, + ) + elif etype == "response.output_item.done": + item = event.item + if item.type == "function_call": + try: + args = ( + json.loads(item.arguments) if item.arguments else {} + ) + except json.JSONDecodeError: + args = {"_raw": item.arguments} + response.add_tool_call( + llm.ToolCall( + tool_call_id=item.call_id, + name=item.name, + arguments=args, + ) + ) + elif etype == "response.completed": + final_response_dict = event.response.model_dump() + if final_response_dict.get("usage"): + usage = final_response_dict["usage"] + if final_response_dict is not None: + response.response_json = remove_dict_none_values( + final_response_dict + ) + else: + completion = await client.responses.create( + model=self.model_name or self.model_id, + input=input_items, + stream=False, + **kwargs, + ) + dumped = completion.model_dump() + response.response_json = remove_dict_none_values(dumped) + usage = dumped.get("usage") + for item in completion.output: + if item.type == "reasoning": + had_reasoning = True + elif item.type == "function_call": + try: + args = json.loads(item.arguments) if item.arguments else {} + except json.JSONDecodeError: + args = {"_raw": item.arguments} + response.add_tool_call( + llm.ToolCall( + tool_call_id=item.call_id, + name=item.name, + arguments=args, + ) + ) + yield StreamEvent( + type="tool_call_name", + chunk=item.name or "", + tool_call_id=item.call_id, + ) + yield StreamEvent( + type="tool_call_args", + chunk=item.arguments or "", + tool_call_id=item.call_id, + ) + elif item.type == "message": + for content in item.content or []: + ctype = getattr(content, "type", None) + if ctype == "output_text" and content.text: + yield StreamEvent(type="text", chunk=content.text) + + self._set_usage_responses(response, usage) + if had_reasoning or ( + usage + and (usage.get("output_tokens_details") or {}).get("reasoning_tokens") + ): + yield StreamEvent(type="reasoning", chunk="", redacted=True) + response._prompt_json = redact_data( + {"input": input_items, "instructions": instructions} + ) + + class Completion(Chat): class Options(SharedOptions): logprobs: Optional[int] = Field( diff --git a/tests/cassettes/test_openai_responses/test_responses_basic_non_streaming.yaml b/tests/cassettes/test_openai_responses/test_responses_basic_non_streaming.yaml new file mode 100644 index 000000000..382945fa4 --- /dev/null +++ b/tests/cassettes/test_openai_responses/test_responses_basic_non_streaming.yaml @@ -0,0 +1,105 @@ +interactions: +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Reply + with exactly: pong"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":false}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '182' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/31U23KjMAx971dkeG46XEPor3R2GAGCeGtsry9pM538+woTHNim+4Z1rGNJ54iv + p90uYl30uos0GlXHTZ7lZZnmCGVWFRAnh6qHqs8OWda2x6RKq6qB/IjFMY/LoivS6HmikM1vbO1C + I4XBOd5qBItdDROWlOUxztI8O3rMWLDOTDmtHBVHujcnNdC+D1o6MdXVAzc4hxnnTAwU+6IjBRRc + UE/5HZ6RS0UHAq7zwwvlw6dRazllCse5D/Qa/zgU7aVWKIDbC4HxS+wxJhayukMLjJt1JhPGatda + Rk2v4yN81tJZ5Wxt5Tt+B62UvG6Bb+lG2SGfehqU3RcvxT6N08M+zvdpFoULGqbn1mnzSxR586OZ + BxSkHc3wH2WbIo4nZZu+aqHJuqQo+6yvEv+eZ7EXhZ4HjYEB78BPEnqwlcKiuBe1LmxDu4wJP23I + 9hdACGlhGe3brw3I5aC0bB4gnoh4yYdDFIDr7SvcjdQJjC+gZ6R5DcJ8kIUCrCX3KBjDqFHq5Gnh + 8RzkP03yId9KSW6Y/apoFchS+MBSBJ2ZdKZetqX2QgU5qbFRWaJsT1i/4+VHTOM04tkMUZqfZo/Q + 0hkpNruCfS+1nwqXH7ceI+PGEfTCHnbHQI/2QiVN1D3DzaYY1GdGTVm27F4Pjs/CkR+kxs3SWhzV + 5Ffn48mt/5tCt9qoshHu55Uz/L156reSz6gbaZifJvmxY268L/2sw0lSfV44Z2UUgLtR6KjqlX3i + EFReo2r+SZCQol02LeqYgYYvvyjn9yB0wMRmz5Pk+Xt89fMIfXoRu3tivOn1399H8Sj+iDbo/xOz + pbXiq3oPYYTObOUeib0DCxP99en6F91lIsYwBgAA + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f745ab1f9b90d35-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 01:53:58 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '679' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=0fpVXxFlMIVJXxNLG4WI.0x_QYZxtQvFBJTKVOrzZh0-1778032438.0716975-1.0.1.1-B1A3sARLTfA0.2DTwlSYNmctFA8F7FeH7BpGZ77WNlbrE_eWKZlWxNet3jFXXDoE12MljXSl7qwyOUfdgytFqz4r4WD.MVk041i0UJCVEw3iHqTOgGlAcuK49npcjApL; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 02:23:58 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_f2c27f9fae8c4bd1bab16d7c2e2968ff + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_openai_responses/test_responses_basic_streaming.yaml b/tests/cassettes/test_openai_responses/test_responses_basic_streaming.yaml new file mode 100644 index 000000000..b62f0bca0 --- /dev/null +++ b/tests/cassettes/test_openai_responses/test_responses_basic_streaming.yaml @@ -0,0 +1,150 @@ +interactions: +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Reply + with exactly: pong"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":true}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '181' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_0e76db07f5b559250169fa9f37236c8192a5eae7ebf8107f93","object":"response","created_at":1778032439,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_0e76db07f5b559250169fa9f37236c8192a5eae7ebf8107f93","object":"response","created_at":1778032439,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"rs_0e76db07f5b559250169fa9f37c0c08192a67605ef2cd4778a","type":"reasoning","encrypted_content":"gAAAAABp-p83R_nd7VU9CJlhj5VcIMvSAdyg8k74Q-WpX6ibEQWXhIcV0h9jbDIin8r-i5XF_5VUV-s86svQsOisfocfvXVIS6QQW4QOuLfQ5zW1DdPnLfc8AomThIgE2OHYS6QS4OlOWl_f6WY217D-NLXAnfrluULi6_GVgNA0ci4EyQJ_DqTpJ6xURh_8IgamBE8fob8lYYUDniTar08Acto9Nq_y71UfwRZp3sjhubDR5r935Gitv9grCqyTmGfjPOYfNlUpIVkAhws9oOwGMH-DqF8G6LJ44c2rY0KZvkowD_0awxkejob5zUCAgM31CwwSCL6w7G7hSZKXjP5wrkPFYrCMd9rbVqS7RuBWkvykeoerLWPnUEe1zBNPYXkh3_QxQTK0l0vNCFDCUqGR94XIH8jQbaTIWUavI74ttjjkCpAuOH-z9GvP7yPYga1R_xwhLsoZZDa8qhZ7H9qdqp0fI1eRnrLRm9UZph576C8lKN_gLrcrxttjKHue9KxtdB0SuEE5DvfS0t5Ly8PhgQMn_JDMMk9LN9xvoE-5Hp2PpW5jq_J4N7uGpb4B-rmuuE7gsLvujgbIN1haJzlQBe_U6B22BR3dE_QH92gUVQpYefi-MgPz1y6fC6BO2BBs1rKfqFrR3uhX4VU0maUuvmepY0Z42Zg2sa0q-5k1uJdi4o9y_XqIxGv0jczfeCSPU3o-hOu_KpclYVEgTmDjDEKYe-_64tE_NECOnRREkr1RlrEo1MyjxI3WWSDKyioPmPYnA_fHOBgAmwzvK8Z3E2COr1Df_dXugONhhrOXVOo1hYljqM25TNjQ025ciaMZOPnIte-WX-fNgeAC7fT6iqFqyQdivwsLUMyplVOvVTSU2-8BuuEVbcgvafBHolZJ14ywpqqycG4ZhMaKvO1ZIlimGfSIWA==","summary":[]},"output_index":0,"sequence_number":2} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"rs_0e76db07f5b559250169fa9f37c0c08192a67605ef2cd4778a","type":"reasoning","encrypted_content":"gAAAAABp-p83DkAVOSy6esnX6kCYlq43wwy2M4QFEa2a4fRuixmbaT1m8OZ2rzFkErePz5D_QOd2G-VqBFzn80vmdls1KOTeI_8GtffOUQqg_VmoJHIwpKAYTn-c6MA5F480FjYfZLOxa3ShyhIuF-_09R3UoAGUHwX2p1cLa10iLrwd1tbXQLJfe1MDt7yoqqAxV7N7P6Cvr6cyBrjvO_wDhC3FZO3h0LuPaAdhkJU5tD3MPoY1vAQZYBqjjxN_2jkzUayx4cs8UeDDEzymFGmO2zsrx4V6YkzvuuuAHTTXBp0eNkThng6Tp-4eWPOmey0ffyN8Uzx9uA_6BaCesHHiSlxrfHGLplPpWp42fK5tKAVblO6HFyg8e16Sq94JTRDGXjadEYS7Lr7j6ndD_1-PJBiV2WbLBXZpEyAc4kqPqm1ldx2W2Piuep_mZqDTPgAqbEWRhQcdO2fchQuWoNphoOdBdFUV6WQ5AwfVQuVeb-fEnjZhACM3xYHoQ472OktkZuNHpWiYKSHrjcs6kanUy4udjVojJG6DIxCCiNYkPqNeP5lrB3swVGbFfuQNgGCFOP8VAIsJOYbRl3sQXbJe0KfRaM5ssChAscLyKSeFiLrscE5vMUH-__wcZenDrKB8ILcV8a3N2O2SEFkDz_9LbvYPpXlWKc8o-88AlX7V2azaT3QVOdeI0T5WJm9pgY8wPzg2hL4nrbBVDEf56ag5klaC9Rm-x2DFgqyjbdPTT0eVayw-xJqMjt2uZDlgtYQXvRuVhieThpC2DS_QLrc3MbnjXj1LD1idGNlzx2QmIJNrjA8HoRusuSkQ4MIQ-1sQH3BuhfcO05Y5ID4s_3Owsr79qQxngvWM2Bqcj3YNrfBhoDNjShzHTc5mDq5DBX603IByUbHqYs2wEvNlRofxn5rqvD5XQlPZ9lyWAME3FfWR4X1Yyho_iLiETidIp7F6khLGyNAe","summary":[]},"output_index":0,"sequence_number":3} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"msg_0e76db07f5b559250169fa9f37db9c81928be5700b5467f40c","type":"message","status":"in_progress","content":[],"phase":"final_answer","role":"assistant"},"output_index":1,"sequence_number":4} + + + event: response.content_part.added + + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_0e76db07f5b559250169fa9f37db9c81928be5700b5467f40c","output_index":1,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":5} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"pong","item_id":"msg_0e76db07f5b559250169fa9f37db9c81928be5700b5467f40c","logprobs":[],"obfuscation":"XaUSm3fYYxcJ","output_index":1,"sequence_number":6} + + + event: response.output_text.done + + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0e76db07f5b559250169fa9f37db9c81928be5700b5467f40c","logprobs":[],"output_index":1,"sequence_number":7,"text":"pong"} + + + event: response.content_part.done + + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_0e76db07f5b559250169fa9f37db9c81928be5700b5467f40c","output_index":1,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"pong"},"sequence_number":8} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"msg_0e76db07f5b559250169fa9f37db9c81928be5700b5467f40c","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"pong"}],"phase":"final_answer","role":"assistant"},"output_index":1,"sequence_number":9} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_0e76db07f5b559250169fa9f37236c8192a5eae7ebf8107f93","object":"response","created_at":1778032439,"status":"completed","background":false,"completed_at":1778032439,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[{"id":"rs_0e76db07f5b559250169fa9f37c0c08192a67605ef2cd4778a","type":"reasoning","encrypted_content":"gAAAAABp-p83SvgfbunuD3kn7u31CtjHU8hbx-JUCVZzDx04cJr1bJU9XkgpMdekH0PSg-GuhLgFJJhlE8484-l2Dhm8HKS-cuXwmipFdckWxvlP8uZ86zHcQOvjcfsARgpB0jNWfmpiiRaL6h5e0lORHoyiY5D9L6lDqcBhJIX_Yh-3Ml5XfvmH_FselvKl19RyZKGsl7z2Pg9T_EvddVVsX1yRYMkusr9iQdXP3jz0yz5LSJc03Pk0oG8502vJUWTlcr_y1meATsylmsWlCJpKVfiMUAOp0qL53DOyflYuwSomq80okIVeRsTf-Fje_y1YNfLo6dKLkVtRcIGCUJuy46bnBW2bF09pQnPd5BHFYI0ePYzoPOAhen3Q6fmvSnhc3OyCx48ZgOOps5W3kMRYDCsNiLnYuLwnIhxGq0lD0cEVRZtuQNtQKmJXTdg1Fy8RxO4S2j_1msvjXKBD4P5H9PbaehKBlkOzwkjhg25Rfc7Lw3wyAWrKGMeEc14RpCAMB1kNGz-aK79dckg-0ihNQ8RkYFF4TZXQ7bczbTIFRfJ-ZFu1pP2-gCLTei03FluGcFQNGY-kZPXeuvzhTCJltIzZ0nay_9KI5Ny_TTBi2MKbpaVDX3oxs3JZ4f0x-s8KjT-RI4ibNKHkF3Da7Fe8VYb21EVyHkEW8Xe5Wcs5OHKxHrAIr2acaov1Md0LJEdB1vd55LPBmO_ZsCXV4-6dCkHlTdV8tD30JyOzFRRjP_rJN4w4JlVqbPGD3m2fB7UtaNy51DOVwzPKrIhEqulBsv72eqSQwad6Nsx5BAi7tt6-KFMPFeeG6GbadyjHAycz1Em4oGQ0_Hh0zvY8tvxSIfaIIP3LEjGkXK7iu9v26b9RbiHU-5JRBDHQO6QqcOZ2PswB725TrTMpmCtjeva5jP4ThRuTt8Zq1ARRMCQrNJgKs2-yFcwSwl1gb3LbAR9ZTJG4MN_z","summary":[]},{"id":"msg_0e76db07f5b559250169fa9f37db9c81928be5700b5467f40c","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"pong"}],"phase":"final_answer","role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":11,"input_tokens_details":{"cached_tokens":0},"output_tokens":16,"output_tokens_details":{"reasoning_tokens":9},"total_tokens":27},"user":null,"metadata":{}},"sequence_number":10} + + + ' + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f745ab7f9720d35-ORD + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 06 May 2026 01:53:59 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '276' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=D0JLG2055c9_Qp1pj16jx7tyQVtYPJ5Jlr8srrADKis-1778032439-1.0.1.1-wSFVNPO_rARMYH4a6F.V29bDImlV73C.ndtzHFwdmMoE0v5MVnLwn7oHEao8S1uWbmzJhYfZSGRIKmWJzogh1mZ0.OmOvQwzRf7q5oEMYc8; + path=/; expires=Wed, 06-May-26 02:23:59 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=vNa5HJF6CAbxVrsu4xC9Rqxc03APkSzlo2KtIijWnHQ-1778032439396-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '39999826' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_f37a6e34cdac4945976e57df6a3ca8fd + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_openai_responses/test_responses_tool_use.yaml b/tests/cassettes/test_openai_responses/test_responses_tool_use.yaml new file mode 100644 index 000000000..ce9086a00 --- /dev/null +++ b/tests/cassettes/test_openai_responses/test_responses_tool_use.yaml @@ -0,0 +1,216 @@ +interactions: +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"What + is 1231 * 2331? Use the multiply tool."}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":false,"tools":[{"type":"function","name":"multiply","description":"Multiply + two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '400' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/5VVyW7bMBC9+ysMnu1Am2Mr5x6KJi2KopeiKQRKGjmMKZHl4sQw/O8dUtbmOAHq + g2HO4zzO8mZ8nM3nhJXkbk4UaJkF6RroJgUooEjW6SYIb9OKplUCQRUWm5CGaR7ROEiSchVBkAcB + WTgKkT9DYToa0Who7YUCaqDMqMPC9XoTxFFyG3lMG2qsdj6FqCUHvNc65bTYbZWwjYurolxDa2ac + s2aLtiMe0SDpAZTzL2EPXEg8IHBqH+4oL56OPQpKCefZWM69oVLw10JTHDIJDeXmgGBwE3iMNR1Z + VoKhjOuxJ2u0UbYwDJMe22v6mglrpDWZETt4CxoheFZQPqWrRQnc5bSVZrm6WS2jILpdBskyikl/ + QVH33NitfQktv31p2gL1ra2KDxpbFmHiG7uOoiqENF7lMazWiX/Ok5iDBE9jG5+nj3qA3+ujB6na + 2hoa4/HjI6GP5C6M4nDxSHL8GcVxeBpuO+Ksjdn/fL7/vHuovlTpge/yX/uH/P6H/PTtZzN4NLT2 + sdWWGyb5gXjghN9/fGEkVUgEfFpu7FirKYlyxbbDlbYjtGfC6qxTdBtYX3KpMFuDlMUTZDs4vIsp + rEhzbhiJkqe2jzgYWjQTPUNVCeWHiIuXc4pE27qmqmPv9a1pBeaAITnqisFEzRrUnmFShnXzUVGs + DzmPnVAwGSwDtXSast4envM38GqG2DCymg7nkSr8vbbq55D3oHKhma8mqaFkth4Gs+3Dk8D4HEyt + EaQH9FsFX4pvaH0JulBMdpX9elbA3LwILEWdg9I3HwilR5xEauyR0qP02iZiWbCGU7sT9YVhFCZr + DGzbRdR9Touxb/5/vrMrLMQtK6ag7KvVx3XxVH/6M/Lunjvv7BFCy5K5clL+fZy7V8rsIgzUkWJ+ + 5bthupg6I2TGxRYrmDuCoDdKP2DppjUo7Gm3ykjJNM159x9gNd3CID/WTBbparN4ax9t5+OwUHAC + y8ExmAj1cj9H8TXgGm8/ve9RG2EoH8BN2A+A1dNhRenRkhqvqdPs9A/gX36dkgcAAA== + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f745b477f340044-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 01:54:23 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '1084' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=z7uNE1DqeNcWOXy7RIEwbyUEWlzdoMxhY9qUkpfgq8E-1778032461.9979239-1.0.1.1-VqVdBmRJortunHk6vz6JIxYW1JMEnm17BsExYD9AGOJBsRDJIWU_K7O0Hnxp_tK6ziVKx4HG7qrTcXV07bZCPdoED.Rg_hKX.CjJv529BuAWX_aerfhFo8Xob0p8.Sfs; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 02:24:23 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_879af2c359a94b25bca83aca9ef2e65f + status: + code: 200 + message: OK +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"What + is 1231 * 2331? Use the multiply tool."},{"type":"function_call","call_id":"call_jKHkLfJf9ylkbYvLbKRpDNTn","name":"multiply","arguments":"{\"a\": + 1231, \"b\": 2331}"},{"type":"function_call_output","call_id":"call_jKHkLfJf9ylkbYvLbKRpDNTn","output":"2869461"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":false,"tools":[{"type":"function","name":"multiply","description":"Multiply + two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '619' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/5VVSY7bMBC8+xWGjoY90OZFAfKEALlPAoESWzYzFMlw8Ywx8N/TIq3NYx9yk7rI + Yi9V5OdiuYwYjb4tIw1GlXES76o8JYe6qYDmWZzsioYUTd7ku/xwSIqsSiAhCdD9NsuKLMujdUch + qz9Q255GCgMhXmsgFmhJOizZ7w9xlua7zGPGEutMt6eWreKA68KmitRvRy2d6PJqCDcQwoxzJo4Y + +8RfDChyAd3tp3AGLhX+IHANB/eUd0fnHgWtZbdTOM59oNHw14GoL6UCQbi9IBi/xB5joicrKVjC + uJnuZMJY7WrLsOhpvCUfpXRWOVta+QZfQSslL2vC53StpMC7mo7KbrYv200ap7tNnG/SLBoWaNId + N90WTsLIq29NaNAw2tYcn052G+d06ydbHHZF2iT5HpoiL2jhz/Ms9qLA84Ax5Agj8GyEHqylsCDG + pKaJzWj7NsGHHXb7BUQIaUnf2tffM5DLo9KyeoB4IuRN0ixZ/nJxTPfLNMPv78vVKl1jmet8l6xW + 0bDpevsaeCJ1IsYn1zDUQ0mEeUd5DbCW3KPEGIZNwCoXPY/nQG1qHC3w+ZhRKUHLCm2CcoMHckPo + zKQzZe+k0g9xGDUW3SqLlPUJyje4PMU0dO0PQonS/BT0g4Y0Usx8BE0jte8Yl++3GiPj2pbonn3w + lSEN2Aum1FE3DGYuMqDPDIuyrPdlQxwPQ0WtSA0zQ1toVadl5+PJrf7b9G65YWYtGf8nqvHrQtdv + KZ9BV9Iw303UKmWuHS+EMIeTxPz84JyV0QCYr87pj2mc8OYeZ0/B1JqpvrM/sEKm+GVp3yW2oq1A + m5dxtSBt8M5t2Yh0EmlxRtpMygtDxLZgD+fxzg93gUmaDL12DBfgIOqZKar/27t4wBJ1lyTTQGeW + 9nndHTX8TYw52j28FROEUMq6dhL+c1q7V8riLg3UkWb+qenMdOc6K1U5uRfiIai8wYpDCGicaX+F + RpQZUvH+7XH+ghvkx8TsAi/y9df45FUY1OMdSMeN8Uyo9+9CcngEPOId3PuM2uKFySfMSTo4wJm5 + W1F7hBLrRXVdXP8BMqG3+wsIAAA= + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f745b4f2f7d0044-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 01:54:24 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '1355' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=n0X3ttAhrLg1o4.WV.ZnmBJk2WZojTqaq1DsDoffi5w-1778032463.221904-1.0.1.1-vn34lp4AgkusR7VdGs7M8S_Gq2eOxBEbOdidg2ZEMx3iopML7ykmJupC_aAHReNHK8vo8.IXAw0zdXeZIhClirFAZoqB1sglnuvDE.x1LcjUY6Be2mdr7Aw67UkJG7Es; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 02:24:24 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_552a72b066df401a8c127fdb2afe042f + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_openai_responses/test_responses_tool_use_streaming.yaml b/tests/cassettes/test_openai_responses/test_responses_tool_use_streaming.yaml new file mode 100644 index 000000000..ca0561d60 --- /dev/null +++ b/tests/cassettes/test_openai_responses/test_responses_tool_use_streaming.yaml @@ -0,0 +1,393 @@ +interactions: +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"What + is 1231 * 2331? Use the multiply tool."}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":true,"tools":[{"type":"function","name":"multiply","description":"Multiply + two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '399' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_044ee23f1de77c370169fa9f50e844819199db824931af4641","object":"response","created_at":1778032464,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Multiply + two numbers.","name":"multiply","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_044ee23f1de77c370169fa9f50e844819199db824931af4641","object":"response","created_at":1778032464,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Multiply + two numbers.","name":"multiply","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","type":"function_call","status":"in_progress","arguments":"","call_id":"call_LHDmxFdi7rwbMbWoZc2CdkHs","name":"multiply"},"output_index":0,"sequence_number":2} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"TSJ31mxTQGPOro","output_index":0,"sequence_number":3} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"a","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"YVpNcmj2Vt5N34c","output_index":0,"sequence_number":4} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"\":","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"rK5gXnCa0e2INo","output_index":0,"sequence_number":5} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"123","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"v10PcJ1qDR8m7","output_index":0,"sequence_number":6} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"1","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"iGpOyjZcI5jzNIp","output_index":0,"sequence_number":7} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":",\"","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"sCuCqOYfD3Hnyi","output_index":0,"sequence_number":8} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"b","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"7tizScBl6S0jyhq","output_index":0,"sequence_number":9} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"\":","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"aTDdMGY2Tk7qnV","output_index":0,"sequence_number":10} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"233","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"kg7VGPdV93SyR","output_index":0,"sequence_number":11} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"1","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"FD8tdtuvZ0Q9NLW","output_index":0,"sequence_number":12} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"}","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"xuOXpmYM4HPWedI","output_index":0,"sequence_number":13} + + + event: response.function_call_arguments.done + + data: {"type":"response.function_call_arguments.done","arguments":"{\"a\":1231,\"b\":2331}","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","output_index":0,"sequence_number":14} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","type":"function_call","status":"completed","arguments":"{\"a\":1231,\"b\":2331}","call_id":"call_LHDmxFdi7rwbMbWoZc2CdkHs","name":"multiply"},"output_index":0,"sequence_number":15} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_044ee23f1de77c370169fa9f50e844819199db824931af4641","object":"response","created_at":1778032464,"status":"completed","background":false,"completed_at":1778032465,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[{"id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","type":"function_call","status":"completed","arguments":"{\"a\":1231,\"b\":2331}","call_id":"call_LHDmxFdi7rwbMbWoZc2CdkHs","name":"multiply"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Multiply + two numbers.","name":"multiply","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":58,"input_tokens_details":{"cached_tokens":0},"output_tokens":23,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":81},"user":null,"metadata":{}},"sequence_number":16} + + + ' + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f745b59588a0d35-ORD + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 06 May 2026 01:54:25 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '262' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=6VXg_BM_1a0kiY0dxVuH8m4VdhuXjNNXVwe8H1X6TnI-1778032464.8521352-1.0.1.1-W9VN6MIUcIqwnCiXpBplvoBRsAGZFB7KzfpnSev8YLeCWnfvp6yEJM5D.ekb1ubddrlBac.xiewMcEfuKRCpRGPnI.AIr3QCYjyGncQtKLAtCRIew34WL.a3qZBuwD0b; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 02:24:25 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '39999493' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_465f47d6541b4fe1963395faa1bf2739 + status: + code: 200 + message: OK +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"What + is 1231 * 2331? Use the multiply tool."},{"type":"function_call","call_id":"call_LHDmxFdi7rwbMbWoZc2CdkHs","name":"multiply","arguments":"{\"a\": + 1231, \"b\": 2331}"},{"type":"function_call_output","call_id":"call_LHDmxFdi7rwbMbWoZc2CdkHs","output":"2869461"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":true,"tools":[{"type":"function","name":"multiply","description":"Multiply + two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '618' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "event: response.created\ndata: {\"type\":\"response.created\",\"response\"\ + :{\"id\":\"resp_06c574f2869625a90169fa9f523b2c81a1a08f987e0980156e\",\"object\"\ + :\"response\",\"created_at\":1778032466,\"status\":\"in_progress\",\"background\"\ + :false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\"\ + :null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\"\ + :null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[],\"\ + parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\"\ + :null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\"\ + :{\"effort\":\"low\",\"summary\":null},\"safety_identifier\":null,\"service_tier\"\ + :\"auto\",\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\"\ + :\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"\ + type\":\"function\",\"description\":\"Multiply two numbers.\",\"name\":\"\ + multiply\",\"parameters\":{\"properties\":{\"a\":{\"type\":\"integer\"},\"\ + b\":{\"type\":\"integer\"}},\"required\":[\"a\",\"b\"],\"type\":\"object\"\ + ,\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\"\ + :0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\"\ + :{}},\"sequence_number\":0}\n\nevent: response.in_progress\ndata: {\"type\"\ + :\"response.in_progress\",\"response\":{\"id\":\"resp_06c574f2869625a90169fa9f523b2c81a1a08f987e0980156e\"\ + ,\"object\":\"response\",\"created_at\":1778032466,\"status\":\"in_progress\"\ + ,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\"\ + :0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\"\ + :null,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\"\ + :null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"\ + previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\"\ + :\"24h\",\"reasoning\":{\"effort\":\"low\",\"summary\":null},\"safety_identifier\"\ + :null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1.0,\"text\"\ + :{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\"\ + :\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Multiply two\ + \ numbers.\",\"name\":\"multiply\",\"parameters\":{\"properties\":{\"a\":{\"\ + type\":\"integer\"},\"b\":{\"type\":\"integer\"}},\"required\":[\"a\",\"b\"\ + ],\"type\":\"object\",\"additionalProperties\":false},\"strict\":true}],\"\ + top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"\ + user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: response.output_item.added\n\ + data: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"phase\"\ + :\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\"\ + :2}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\"\ + ,\"content_index\":0,\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"\ + logprobs\":[],\"text\":\"\"},\"sequence_number\":3}\n\nevent: response.output_text.delta\n\ + data: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\"\ + :\"123\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"logprobs\":[],\"obfuscation\":\"PsjcK87GfnMGs\",\"output_index\":0,\"sequence_number\"\ + :4}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\"1\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"logprobs\":[],\"obfuscation\":\"wcAoCtBmb6T7zWp\",\"output_index\":0,\"\ + sequence_number\":5}\n\nevent: response.output_text.delta\ndata: {\"type\"\ + :\"response.output_text.delta\",\"content_index\":0,\"delta\":\" \xD7\",\"\ + item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\",\"logprobs\"\ + :[],\"obfuscation\":\"5cOLfvSirBOxck\",\"output_index\":0,\"sequence_number\"\ + :6}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\" \",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"logprobs\":[],\"obfuscation\":\"yDlX8TgyBiqvfBc\",\"output_index\":0,\"\ + sequence_number\":7}\n\nevent: response.output_text.delta\ndata: {\"type\"\ + :\"response.output_text.delta\",\"content_index\":0,\"delta\":\"233\",\"item_id\"\ + :\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\",\"logprobs\":[],\"\ + obfuscation\":\"lwWcyMe8rNUNP\",\"output_index\":0,\"sequence_number\":8}\n\ + \nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\"1\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"logprobs\":[],\"obfuscation\":\"N6xkEhKWhnK9zYZ\",\"output_index\":0,\"\ + sequence_number\":9}\n\nevent: response.output_text.delta\ndata: {\"type\"\ + :\"response.output_text.delta\",\"content_index\":0,\"delta\":\" =\",\"item_id\"\ + :\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\",\"logprobs\":[],\"\ + obfuscation\":\"fw7O0adbH3ppH9\",\"output_index\":0,\"sequence_number\":10}\n\ + \nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\" **\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"logprobs\":[],\"obfuscation\":\"MYjlLyYLXMlda\",\"output_index\":0,\"sequence_number\"\ + :11}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\"2\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"logprobs\":[],\"obfuscation\":\"lR21X59BeOsCbaf\",\"output_index\":0,\"\ + sequence_number\":12}\n\nevent: response.output_text.delta\ndata: {\"type\"\ + :\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\"\ + :\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\",\"logprobs\":[],\"\ + obfuscation\":\"MFmjQhCxnhZoDZJ\",\"output_index\":0,\"sequence_number\":13}\n\ + \nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\"869\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"logprobs\":[],\"obfuscation\":\"wLjwhrxjSc1uy\",\"output_index\":0,\"sequence_number\"\ + :14}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"logprobs\":[],\"obfuscation\":\"LfxW516uVZ5YUor\",\"output_index\":0,\"\ + sequence_number\":15}\n\nevent: response.output_text.delta\ndata: {\"type\"\ + :\"response.output_text.delta\",\"content_index\":0,\"delta\":\"461\",\"item_id\"\ + :\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\",\"logprobs\":[],\"\ + obfuscation\":\"oFk25jkAOAbD6\",\"output_index\":0,\"sequence_number\":16}\n\ + \nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\"**\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"logprobs\":[],\"obfuscation\":\"F0z6oUjKY4ZGlI\",\"output_index\":0,\"\ + sequence_number\":17}\n\nevent: response.output_text.done\ndata: {\"type\"\ + :\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"logprobs\":[],\"output_index\":0,\"sequence_number\":18,\"text\":\"1231\ + \ \xD7 2331 = **2,869,461**\"}\n\nevent: response.content_part.done\ndata:\ + \ {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\"\ + :\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\",\"output_index\"\ + :0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"\ + text\":\"1231 \xD7 2331 = **2,869,461**\"},\"sequence_number\":19}\n\nevent:\ + \ response.output_item.done\ndata: {\"type\":\"response.output_item.done\"\ + ,\"item\":{\"id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\"\ + ,\"annotations\":[],\"logprobs\":[],\"text\":\"1231 \xD7 2331 = **2,869,461**\"\ + }],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"\ + sequence_number\":20}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\"\ + ,\"response\":{\"id\":\"resp_06c574f2869625a90169fa9f523b2c81a1a08f987e0980156e\"\ + ,\"object\":\"response\",\"created_at\":1778032466,\"status\":\"completed\"\ + ,\"background\":false,\"completed_at\":1778032467,\"error\":null,\"frequency_penalty\"\ + :0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\"\ + :null,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\"\ + :null,\"output\":[{\"id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\"\ + ,\"annotations\":[],\"logprobs\":[],\"text\":\"1231 \xD7 2331 = **2,869,461**\"\ + }],\"phase\":\"final_answer\",\"role\":\"assistant\"}],\"parallel_tool_calls\"\ + :true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\"\ + :null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"effort\":\"low\"\ + ,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\"\ + ,\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"\ + },\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\"\ + :\"function\",\"description\":\"Multiply two numbers.\",\"name\":\"multiply\"\ + ,\"parameters\":{\"properties\":{\"a\":{\"type\":\"integer\"},\"b\":{\"type\"\ + :\"integer\"}},\"required\":[\"a\",\"b\"],\"type\":\"object\",\"additionalProperties\"\ + :false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\"\ + :\"disabled\",\"usage\":{\"input_tokens\":94,\"input_tokens_details\":{\"\ + cached_tokens\":0},\"output_tokens\":18,\"output_tokens_details\":{\"reasoning_tokens\"\ + :0},\"total_tokens\":112},\"user\":null,\"metadata\":{}},\"sequence_number\"\ + :21}\n\n" + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f745b61ac4a0044-ORD + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 06 May 2026 01:54:26 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '310' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=FINbeYv4P8EmjyVhAWYLQKgBylkGZZfMLrWBueXEPEU-1778032466.1842103-1.0.1.1-Z5aNN38FYUcK1gh7SXuH81x1g73cX2Etv1nZkiRDxuaShelUUtL7xi_DUs3aRin22APa4.qbzj3.HOslue0ntVZ2.Bqyt5gPyMnFk3o9pVo8LnBCJgKKR7BFDhbOtKS2; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 02:24:26 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '39999457' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_87194de4a3a744a78ed7904a96cd1e5e + status: + code: 200 + message: OK +version: 1 diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py new file mode 100644 index 000000000..84edb0f2c --- /dev/null +++ b/tests/test_openai_responses.py @@ -0,0 +1,219 @@ +"""Tests for the /v1/responses code path in the default OpenAI plugin.""" + +import json +import os + +import llm +import pytest + +API_KEY = os.environ.get("PYTEST_OPENAI_API_KEY", None) or "badkey" + + +def test_responses_model_is_registered(): + model = llm.get_model("gpt-5.5") + assert "Responses" in type(model).__name__ + # The chat_completions opt-out option must be exposed. + assert "chat_completions" in model.Options.model_fields + + +def test_chat_completions_opt_out_dispatches_to_chat(httpx_mock): + """When chat_completions=1 is passed, the request must hit + /v1/chat/completions, not /v1/responses.""" + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "id": "chatcmpl-x", + "object": "chat.completion", + "model": "gpt-5.5", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hi from chat"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + }, + headers={"Content-Type": "application/json"}, + ) + model = llm.get_model("gpt-5.5") + response = model.prompt("hello", stream=False, chat_completions=True, key="test") + assert response.text() == "hi from chat" + + +def test_default_routes_to_responses_endpoint(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_test_1", + "object": "response", + "created_at": 1, + "model": "gpt-5.5", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [ + {"type": "output_text", "text": "hi from responses", "annotations": []} + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + }, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + model = llm.get_model("gpt-5.5") + response = model.prompt("hello", stream=False, key="test") + assert response.text() == "hi from responses" + # Ensure we sent to the right endpoint + requests = [r for r in httpx_mock.get_requests()] + assert any("/v1/responses" in str(r.url) for r in requests) + + +def test_responses_input_translation(): + """Unit-test the message-to-input translator without hitting the API.""" + from llm.parts import ( + Message, + TextPart, + ToolCallPart, + ToolResultPart, + AttachmentPart, + ) + + model = llm.get_model("gpt-5.5") + + class FakePrompt: + messages = [ + Message(role="system", parts=[TextPart(text="be brief")]), + Message(role="user", parts=[TextPart(text="2 + 2?")]), + Message( + role="assistant", + parts=[ + ToolCallPart( + name="add", + arguments={"a": 2, "b": 2}, + tool_call_id="call_abc", + ) + ], + ), + Message( + role="tool", + parts=[ + ToolResultPart( + name="add", output="4", tool_call_id="call_abc" + ) + ], + ), + ] + + items, instructions = model._build_responses_input(FakePrompt()) + assert instructions == "be brief" + # First user message is a plain string content + assert items[0] == {"role": "user", "content": "2 + 2?"} + # function_call from assistant + assert items[1]["type"] == "function_call" + assert items[1]["call_id"] == "call_abc" + assert items[1]["name"] == "add" + assert json.loads(items[1]["arguments"]) == {"a": 2, "b": 2} + # tool result + assert items[2] == { + "type": "function_call_output", + "call_id": "call_abc", + "output": "4", + } + + +def test_responses_kwargs_packs_reasoning_and_verbosity(): + model = llm.get_model("gpt-5.5") + options = model.Options(reasoning_effort="low", verbosity="low") + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + kwargs = model._build_responses_kwargs(p, stream=False) + assert kwargs["reasoning"] == {"effort": "low"} + assert kwargs["text"]["verbosity"] == "low" + + +@pytest.mark.vcr +def test_responses_basic_non_streaming(vcr): + model = llm.get_model("gpt-5.5") + response = model.prompt( + "Reply with exactly: pong", + stream=False, + reasoning_effort="low", + key=API_KEY, + ) + text = response.text() + assert "pong" in text.lower() + # response_json should reflect the Responses API shape + assert response.response_json["object"] == "response" + + +@pytest.mark.vcr +def test_responses_basic_streaming(vcr): + model = llm.get_model("gpt-5.5") + response = model.prompt( + "Reply with exactly: pong", + reasoning_effort="low", + key=API_KEY, + ) + chunks = list(response) + text = "".join(chunks) + assert "pong" in text.lower() + + +@pytest.mark.vcr +def test_responses_tool_use(vcr): + model = llm.get_model("gpt-5.5") + + def multiply(a: int, b: int) -> int: + "Multiply two numbers." + return a * b + + chain = model.chain( + "What is 1231 * 2331? Use the multiply tool.", + tools=[multiply], + stream=False, + options={"reasoning_effort": "low"}, + key=API_KEY, + ) + output = chain.text() + assert "2869461" in output.replace(",", "") + first, second = chain._responses + assert first.tool_calls()[0].name == "multiply" + assert first.tool_calls()[0].arguments == {"a": 1231, "b": 2331} + assert second.prompt.tool_results[0].output == "2869461" + + +@pytest.mark.vcr +def test_responses_tool_use_streaming(vcr): + model = llm.get_model("gpt-5.5") + + def multiply(a: int, b: int) -> int: + "Multiply two numbers." + return a * b + + chain = model.chain( + "What is 1231 * 2331? Use the multiply tool.", + tools=[multiply], + options={"reasoning_effort": "low"}, + key=API_KEY, + ) + output = "".join(chain) + assert "2869461" in output.replace(",", "") + first, second = chain._responses + assert first.tool_calls()[0].arguments == {"a": 1231, "b": 2331} From c7464eeb97968b308d6e08c3aca73b83599ac0ff Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 03:15:21 +0000 Subject: [PATCH 074/258] Round-trip encrypted reasoning across tool calls When the Responses API returns a reasoning item alongside function calls, capture its opaque id + encrypted_content as provider_metadata on the resulting ReasoningPart. _build_responses_input already echoed that metadata back as a reasoning input item on the next turn - now the output side actually populates it. This preserves the model's hidden chain of thought across the tool round-trip. Without it, GPT-5-class models silently lose ~3% on SWE-bench (per OpenAI) when used with tools. Adds a dedicated VCR test that asserts the encrypted_content captured on the first turn appears verbatim in the second turn's outgoing request body. --- llm/default_plugins/openai_models.py | 61 ++- .../test_responses_basic_non_streaming.yaml | 42 ++- .../test_responses_basic_streaming.yaml | 46 +-- ...onses_round_trips_encrypted_reasoning.yaml | 356 ++++++++++++++++++ .../test_responses_tool_use.yaml | 86 ++--- .../test_responses_tool_use_streaming.yaml | 144 +++---- tests/test_openai_responses.py | 53 +++ 7 files changed, 612 insertions(+), 176 deletions(-) create mode 100644 tests/cassettes/test_openai_responses/test_responses_round_trips_encrypted_reasoning.yaml diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 3d62a8f95..2f2ca798a 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1407,6 +1407,38 @@ def _set_usage_responses(self, response, usage): input=input_tokens, output=output_tokens, details=details or None ) + def _reasoning_event(self, item): + """Build a redacted-reasoning StreamEvent that carries the opaque + ``id`` and ``encrypted_content`` from a Responses-API reasoning + item. Echoing this metadata back on the next request via + ``_build_responses_input`` lets the model pick up its prior chain + of thought - critical for tool-using reasoning models, since + without it the model loses ~3% on SWE-bench (per OpenAI).""" + rid = getattr(item, "id", None) + enc = getattr(item, "encrypted_content", None) + summary = getattr(item, "summary", None) + meta: Dict[str, Any] = {} + if rid: + meta["id"] = rid + if enc: + meta["encrypted_content"] = enc + if summary: + # ``summary`` is a list of {type:"summary_text", text:"..."} + # objects when reasoning summaries are enabled. + try: + meta["summary"] = [ + s.model_dump() if hasattr(s, "model_dump") else dict(s) + for s in summary + ] + except Exception: + meta["summary"] = list(summary) + return StreamEvent( + type="reasoning", + chunk="", + redacted=True, + provider_metadata={"openai": meta} if meta else None, + ) + class Responses(_SharedResponses, KeyModel): needs_key = "openai" @@ -1519,8 +1551,6 @@ def execute( chunk=item.name or "", tool_call_id=item.call_id, ) - elif item.type == "reasoning": - had_reasoning = True elif etype == "response.output_text.delta": yield StreamEvent(type="text", chunk=event.delta or "") elif etype == "response.function_call_arguments.delta": @@ -1534,7 +1564,10 @@ def execute( ) elif etype == "response.output_item.done": item = event.item - if item.type == "function_call": + if item.type == "reasoning": + had_reasoning = True + yield self._reasoning_event(item) + elif item.type == "function_call": try: args = ( json.loads(item.arguments) if item.arguments else {} @@ -1569,6 +1602,7 @@ def execute( for item in completion.output: if item.type == "reasoning": had_reasoning = True + yield self._reasoning_event(item) elif item.type == "function_call": try: args = json.loads(item.arguments) if item.arguments else {} @@ -1598,9 +1632,11 @@ def execute( yield StreamEvent(type="text", chunk=content.text) self._set_usage_responses(response, usage) - if had_reasoning or ( - usage - and (usage.get("output_tokens_details") or {}).get("reasoning_tokens") + # Fallback: usage said reasoning happened but the API gave us no + # reasoning items to harvest encrypted_content from. Emit the + # opaque "reasoning happened" marker for UI / token accounting. + if not had_reasoning and usage and ( + (usage.get("output_tokens_details") or {}).get("reasoning_tokens") ): yield StreamEvent(type="reasoning", chunk="", redacted=True) response._prompt_json = redact_data( @@ -1720,8 +1756,6 @@ async def execute( chunk=item.name or "", tool_call_id=item.call_id, ) - elif item.type == "reasoning": - had_reasoning = True elif etype == "response.output_text.delta": yield StreamEvent(type="text", chunk=event.delta or "") elif etype == "response.function_call_arguments.delta": @@ -1735,7 +1769,10 @@ async def execute( ) elif etype == "response.output_item.done": item = event.item - if item.type == "function_call": + if item.type == "reasoning": + had_reasoning = True + yield self._reasoning_event(item) + elif item.type == "function_call": try: args = ( json.loads(item.arguments) if item.arguments else {} @@ -1770,6 +1807,7 @@ async def execute( for item in completion.output: if item.type == "reasoning": had_reasoning = True + yield self._reasoning_event(item) elif item.type == "function_call": try: args = json.loads(item.arguments) if item.arguments else {} @@ -1799,9 +1837,8 @@ async def execute( yield StreamEvent(type="text", chunk=content.text) self._set_usage_responses(response, usage) - if had_reasoning or ( - usage - and (usage.get("output_tokens_details") or {}).get("reasoning_tokens") + if not had_reasoning and usage and ( + (usage.get("output_tokens_details") or {}).get("reasoning_tokens") ): yield StreamEvent(type="reasoning", chunk="", redacted=True) response._prompt_json = redact_data( diff --git a/tests/cassettes/test_openai_responses/test_responses_basic_non_streaming.yaml b/tests/cassettes/test_openai_responses/test_responses_basic_non_streaming.yaml index 382945fa4..1faaee3df 100644 --- a/tests/cassettes/test_openai_responses/test_responses_basic_non_streaming.yaml +++ b/tests/cassettes/test_openai_responses/test_responses_basic_non_streaming.yaml @@ -40,35 +40,35 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAA/31U23KjMAx971dkeG46XEPor3R2GAGCeGtsry9pM538+woTHNim+4Z1rGNJ54iv - p90uYl30uos0GlXHTZ7lZZnmCGVWFRAnh6qHqs8OWda2x6RKq6qB/IjFMY/LoivS6HmikM1vbO1C - I4XBOd5qBItdDROWlOUxztI8O3rMWLDOTDmtHBVHujcnNdC+D1o6MdXVAzc4hxnnTAwU+6IjBRRc - UE/5HZ6RS0UHAq7zwwvlw6dRazllCse5D/Qa/zgU7aVWKIDbC4HxS+wxJhayukMLjJt1JhPGatda - Rk2v4yN81tJZ5Wxt5Tt+B62UvG6Bb+lG2SGfehqU3RcvxT6N08M+zvdpFoULGqbn1mnzSxR586OZ - BxSkHc3wH2WbIo4nZZu+aqHJuqQo+6yvEv+eZ7EXhZ4HjYEB78BPEnqwlcKiuBe1LmxDu4wJP23I - 9hdACGlhGe3brw3I5aC0bB4gnoh4yYdDFIDr7SvcjdQJjC+gZ6R5DcJ8kIUCrCX3KBjDqFHq5Gnh - 8RzkP03yId9KSW6Y/apoFchS+MBSBJ2ZdKZetqX2QgU5qbFRWaJsT1i/4+VHTOM04tkMUZqfZo/Q - 0hkpNruCfS+1nwqXH7ceI+PGEfTCHnbHQI/2QiVN1D3DzaYY1GdGTVm27F4Pjs/CkR+kxs3SWhzV - 5Ffn48mt/5tCt9qoshHu55Uz/L156reSz6gbaZifJvmxY268L/2sw0lSfV44Z2UUgLtR6KjqlX3i - EFReo2r+SZCQol02LeqYgYYvvyjn9yB0wMRmz5Pk+Xt89fMIfXoRu3tivOn1399H8Sj+iDbo/xOz - pbXiq3oPYYTObOUeib0DCxP99en6F91lIsYwBgAA + H4sIAAAAAAAAA3RU247jIAx971dEPE9HSS9J2l8ZrZADTsuWAAumM9Wo/74KadJmpn1LfPDB9vHh + e5FlTEm2z5jH4HheS9mut0W1LcpNJcu8KHctNEVTQwWiLiCvi2aHq02eY121Tblmbz2Fbf6ioJHG + moBDXHgEQsmhx4qqqvN1VVRlwgIBxdDnCNs5jYRySGpAnA7eRtPX1YIOOISV1soc2D77XmRZljEH + F/R9vsQzauvQs0WWXYeLR8ofV1cJRe9tn2mi1inQevwX0YgLd2hA04Xts/w9T5gyIxmXSKB0eMxU + JpCPgpQ1s3gHX9xGcpE42RP+BslazQXoOV1nJeq+p4Oj5fZ9u1zlq3KZb5ar26T7Ax766x7ThpvY + PvtIoxkGNEnbhcNrZXf1qk7KNtiWuw0IWcntusjLdF9ioYvDxIMhwAHvwCsJEyisITT3oh4Lm9GO + Y8IvmrLTATDGEoyj/fgzA7U9OG+bJ0gi2mfMWXNgE3C9fU1nmTtCSAW0yoDmYMIn+nv93uqEQggq + EBgauHqexMEceNAa9VxK8nHYV+cxoBH4ZKWcx7OyMfDRLTwJNcnpvO0ccQHiiPyEl5eYx37EwzKw + 1eY47IhHCNbMvIJta32airaftx5ZiF0HfmSfvBOgRbpwJXvqVuHMKQH9WQnkpEbvtRD1IBwLZD3O + TEvYuX5fY4oXt/5vCt1qa63v4P7/sBnp3DD1W8ln9I0NKk2TdShV7O6mH3Q4WiUG4SJZNgH3RWFk + HX9Yn3wKuqTRrh4CPhoxOo1JFaDR4xMVkw+mDpSZ+bwo3n7HHx6Pqc8korwn5rNefz4f22fxZ7ST + /q+YyRLoh3rLaYQxzOXukEACQU9/XVz/AwAA//8DAEAnS6QwBgAA headers: alt-svc: - h3=":443"; ma=86400 cf-cache-status: - DYNAMIC cf-ray: - - 9f745ab1f9b90d35-ORD + - 9f74ce61c87211fd-ORD content-encoding: - gzip content-type: - application/json date: - - Wed, 06 May 2026 01:53:58 GMT + - Wed, 06 May 2026 03:12:57 GMT openai-organization: - user-r3e61fpak04cbaokp5buoae4 openai-processing-ms: - - '679' + - '1036' openai-project: - proj_2Y4H7T2XPShczTQx5fqMXdNp openai-version: @@ -76,13 +76,17 @@ interactions: server: - cloudflare set-cookie: - - __cf_bm=0fpVXxFlMIVJXxNLG4WI.0x_QYZxtQvFBJTKVOrzZh0-1778032438.0716975-1.0.1.1-B1A3sARLTfA0.2DTwlSYNmctFA8F7FeH7BpGZ77WNlbrE_eWKZlWxNet3jFXXDoE12MljXSl7qwyOUfdgytFqz4r4WD.MVk041i0UJCVEw3iHqTOgGlAcuK49npcjApL; - HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 - 02:23:58 GMT + - __cf_bm=UYuMLYCou_tA0.A8bkTTDs0INBmHlG2j7U9W9gdY8rQ-1778037177-1.0.1.1-5g.mxYNPGVDfBz2paukjljQbRq5PH1x6hIXpFFHvfnbkZxklDSJMF7lUnwNKNv48F1tAMpQxDGkwbCVoTbZJsbFM324N18q7IanrQ5hcaB8; + path=/; expires=Wed, 06-May-26 03:42:57 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=j9wkRiR3FXF4b0PnFv7DnktylnlM._0WSw8ZUpHPQmQ-1778037177704-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None strict-transport-security: - max-age=31536000; includeSubDomains; preload transfer-encoding: - chunked + vary: + - Accept-Encoding x-content-type-options: - nosniff x-ratelimit-limit-requests: @@ -98,7 +102,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_f2c27f9fae8c4bd1bab16d7c2e2968ff + - req_c6713bbdac8d4639a5ea09cb6c5eb5a9 status: code: 200 message: OK diff --git a/tests/cassettes/test_openai_responses/test_responses_basic_streaming.yaml b/tests/cassettes/test_openai_responses/test_responses_basic_streaming.yaml index b62f0bca0..2f7e820ea 100644 --- a/tests/cassettes/test_openai_responses/test_responses_basic_streaming.yaml +++ b/tests/cassettes/test_openai_responses/test_responses_basic_streaming.yaml @@ -41,57 +41,47 @@ interactions: body: string: 'event: response.created - data: {"type":"response.created","response":{"id":"resp_0e76db07f5b559250169fa9f37236c8192a5eae7ebf8107f93","object":"response","created_at":1778032439,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + data: {"type":"response.created","response":{"id":"resp_00592e63e61b66660169fab1b9f8e481a2b321356198d7ac1b","object":"response","created_at":1778037177,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} event: response.in_progress - data: {"type":"response.in_progress","response":{"id":"resp_0e76db07f5b559250169fa9f37236c8192a5eae7ebf8107f93","object":"response","created_at":1778032439,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + data: {"type":"response.in_progress","response":{"id":"resp_00592e63e61b66660169fab1b9f8e481a2b321356198d7ac1b","object":"response","created_at":1778037177,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} event: response.output_item.added - data: {"type":"response.output_item.added","item":{"id":"rs_0e76db07f5b559250169fa9f37c0c08192a67605ef2cd4778a","type":"reasoning","encrypted_content":"gAAAAABp-p83R_nd7VU9CJlhj5VcIMvSAdyg8k74Q-WpX6ibEQWXhIcV0h9jbDIin8r-i5XF_5VUV-s86svQsOisfocfvXVIS6QQW4QOuLfQ5zW1DdPnLfc8AomThIgE2OHYS6QS4OlOWl_f6WY217D-NLXAnfrluULi6_GVgNA0ci4EyQJ_DqTpJ6xURh_8IgamBE8fob8lYYUDniTar08Acto9Nq_y71UfwRZp3sjhubDR5r935Gitv9grCqyTmGfjPOYfNlUpIVkAhws9oOwGMH-DqF8G6LJ44c2rY0KZvkowD_0awxkejob5zUCAgM31CwwSCL6w7G7hSZKXjP5wrkPFYrCMd9rbVqS7RuBWkvykeoerLWPnUEe1zBNPYXkh3_QxQTK0l0vNCFDCUqGR94XIH8jQbaTIWUavI74ttjjkCpAuOH-z9GvP7yPYga1R_xwhLsoZZDa8qhZ7H9qdqp0fI1eRnrLRm9UZph576C8lKN_gLrcrxttjKHue9KxtdB0SuEE5DvfS0t5Ly8PhgQMn_JDMMk9LN9xvoE-5Hp2PpW5jq_J4N7uGpb4B-rmuuE7gsLvujgbIN1haJzlQBe_U6B22BR3dE_QH92gUVQpYefi-MgPz1y6fC6BO2BBs1rKfqFrR3uhX4VU0maUuvmepY0Z42Zg2sa0q-5k1uJdi4o9y_XqIxGv0jczfeCSPU3o-hOu_KpclYVEgTmDjDEKYe-_64tE_NECOnRREkr1RlrEo1MyjxI3WWSDKyioPmPYnA_fHOBgAmwzvK8Z3E2COr1Df_dXugONhhrOXVOo1hYljqM25TNjQ025ciaMZOPnIte-WX-fNgeAC7fT6iqFqyQdivwsLUMyplVOvVTSU2-8BuuEVbcgvafBHolZJ14ywpqqycG4ZhMaKvO1ZIlimGfSIWA==","summary":[]},"output_index":0,"sequence_number":2} - - - event: response.output_item.done - - data: {"type":"response.output_item.done","item":{"id":"rs_0e76db07f5b559250169fa9f37c0c08192a67605ef2cd4778a","type":"reasoning","encrypted_content":"gAAAAABp-p83DkAVOSy6esnX6kCYlq43wwy2M4QFEa2a4fRuixmbaT1m8OZ2rzFkErePz5D_QOd2G-VqBFzn80vmdls1KOTeI_8GtffOUQqg_VmoJHIwpKAYTn-c6MA5F480FjYfZLOxa3ShyhIuF-_09R3UoAGUHwX2p1cLa10iLrwd1tbXQLJfe1MDt7yoqqAxV7N7P6Cvr6cyBrjvO_wDhC3FZO3h0LuPaAdhkJU5tD3MPoY1vAQZYBqjjxN_2jkzUayx4cs8UeDDEzymFGmO2zsrx4V6YkzvuuuAHTTXBp0eNkThng6Tp-4eWPOmey0ffyN8Uzx9uA_6BaCesHHiSlxrfHGLplPpWp42fK5tKAVblO6HFyg8e16Sq94JTRDGXjadEYS7Lr7j6ndD_1-PJBiV2WbLBXZpEyAc4kqPqm1ldx2W2Piuep_mZqDTPgAqbEWRhQcdO2fchQuWoNphoOdBdFUV6WQ5AwfVQuVeb-fEnjZhACM3xYHoQ472OktkZuNHpWiYKSHrjcs6kanUy4udjVojJG6DIxCCiNYkPqNeP5lrB3swVGbFfuQNgGCFOP8VAIsJOYbRl3sQXbJe0KfRaM5ssChAscLyKSeFiLrscE5vMUH-__wcZenDrKB8ILcV8a3N2O2SEFkDz_9LbvYPpXlWKc8o-88AlX7V2azaT3QVOdeI0T5WJm9pgY8wPzg2hL4nrbBVDEf56ag5klaC9Rm-x2DFgqyjbdPTT0eVayw-xJqMjt2uZDlgtYQXvRuVhieThpC2DS_QLrc3MbnjXj1LD1idGNlzx2QmIJNrjA8HoRusuSkQ4MIQ-1sQH3BuhfcO05Y5ID4s_3Owsr79qQxngvWM2Bqcj3YNrfBhoDNjShzHTc5mDq5DBX603IByUbHqYs2wEvNlRofxn5rqvD5XQlPZ9lyWAME3FfWR4X1Yyho_iLiETidIp7F6khLGyNAe","summary":[]},"output_index":0,"sequence_number":3} - - - event: response.output_item.added - - data: {"type":"response.output_item.added","item":{"id":"msg_0e76db07f5b559250169fa9f37db9c81928be5700b5467f40c","type":"message","status":"in_progress","content":[],"phase":"final_answer","role":"assistant"},"output_index":1,"sequence_number":4} + data: {"type":"response.output_item.added","item":{"id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","type":"message","status":"in_progress","content":[],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":2} event: response.content_part.added - data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_0e76db07f5b559250169fa9f37db9c81928be5700b5467f40c","output_index":1,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":5} + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} event: response.output_text.delta - data: {"type":"response.output_text.delta","content_index":0,"delta":"pong","item_id":"msg_0e76db07f5b559250169fa9f37db9c81928be5700b5467f40c","logprobs":[],"obfuscation":"XaUSm3fYYxcJ","output_index":1,"sequence_number":6} + data: {"type":"response.output_text.delta","content_index":0,"delta":"pong","item_id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","logprobs":[],"obfuscation":"mZhSFjZXJr0X","output_index":0,"sequence_number":4} event: response.output_text.done - data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0e76db07f5b559250169fa9f37db9c81928be5700b5467f40c","logprobs":[],"output_index":1,"sequence_number":7,"text":"pong"} + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","logprobs":[],"output_index":0,"sequence_number":5,"text":"pong"} event: response.content_part.done - data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_0e76db07f5b559250169fa9f37db9c81928be5700b5467f40c","output_index":1,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"pong"},"sequence_number":8} + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"pong"},"sequence_number":6} event: response.output_item.done - data: {"type":"response.output_item.done","item":{"id":"msg_0e76db07f5b559250169fa9f37db9c81928be5700b5467f40c","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"pong"}],"phase":"final_answer","role":"assistant"},"output_index":1,"sequence_number":9} + data: {"type":"response.output_item.done","item":{"id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"pong"}],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":7} event: response.completed - data: {"type":"response.completed","response":{"id":"resp_0e76db07f5b559250169fa9f37236c8192a5eae7ebf8107f93","object":"response","created_at":1778032439,"status":"completed","background":false,"completed_at":1778032439,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[{"id":"rs_0e76db07f5b559250169fa9f37c0c08192a67605ef2cd4778a","type":"reasoning","encrypted_content":"gAAAAABp-p83SvgfbunuD3kn7u31CtjHU8hbx-JUCVZzDx04cJr1bJU9XkgpMdekH0PSg-GuhLgFJJhlE8484-l2Dhm8HKS-cuXwmipFdckWxvlP8uZ86zHcQOvjcfsARgpB0jNWfmpiiRaL6h5e0lORHoyiY5D9L6lDqcBhJIX_Yh-3Ml5XfvmH_FselvKl19RyZKGsl7z2Pg9T_EvddVVsX1yRYMkusr9iQdXP3jz0yz5LSJc03Pk0oG8502vJUWTlcr_y1meATsylmsWlCJpKVfiMUAOp0qL53DOyflYuwSomq80okIVeRsTf-Fje_y1YNfLo6dKLkVtRcIGCUJuy46bnBW2bF09pQnPd5BHFYI0ePYzoPOAhen3Q6fmvSnhc3OyCx48ZgOOps5W3kMRYDCsNiLnYuLwnIhxGq0lD0cEVRZtuQNtQKmJXTdg1Fy8RxO4S2j_1msvjXKBD4P5H9PbaehKBlkOzwkjhg25Rfc7Lw3wyAWrKGMeEc14RpCAMB1kNGz-aK79dckg-0ihNQ8RkYFF4TZXQ7bczbTIFRfJ-ZFu1pP2-gCLTei03FluGcFQNGY-kZPXeuvzhTCJltIzZ0nay_9KI5Ny_TTBi2MKbpaVDX3oxs3JZ4f0x-s8KjT-RI4ibNKHkF3Da7Fe8VYb21EVyHkEW8Xe5Wcs5OHKxHrAIr2acaov1Md0LJEdB1vd55LPBmO_ZsCXV4-6dCkHlTdV8tD30JyOzFRRjP_rJN4w4JlVqbPGD3m2fB7UtaNy51DOVwzPKrIhEqulBsv72eqSQwad6Nsx5BAi7tt6-KFMPFeeG6GbadyjHAycz1Em4oGQ0_Hh0zvY8tvxSIfaIIP3LEjGkXK7iu9v26b9RbiHU-5JRBDHQO6QqcOZ2PswB725TrTMpmCtjeva5jP4ThRuTt8Zq1ARRMCQrNJgKs2-yFcwSwl1gb3LbAR9ZTJG4MN_z","summary":[]},{"id":"msg_0e76db07f5b559250169fa9f37db9c81928be5700b5467f40c","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"pong"}],"phase":"final_answer","role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":11,"input_tokens_details":{"cached_tokens":0},"output_tokens":16,"output_tokens_details":{"reasoning_tokens":9},"total_tokens":27},"user":null,"metadata":{}},"sequence_number":10} + data: {"type":"response.completed","response":{"id":"resp_00592e63e61b66660169fab1b9f8e481a2b321356198d7ac1b","object":"response","created_at":1778037177,"status":"completed","background":false,"completed_at":1778037178,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[{"id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"pong"}],"phase":"final_answer","role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":11,"input_tokens_details":{"cached_tokens":0},"output_tokens":5,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":16},"user":null,"metadata":{}},"sequence_number":8} ' @@ -101,15 +91,15 @@ interactions: cf-cache-status: - DYNAMIC cf-ray: - - 9f745ab7f9720d35-ORD + - 9f74ce69fcf0cb75-DFW content-type: - text/event-stream; charset=utf-8 date: - - Wed, 06 May 2026 01:53:59 GMT + - Wed, 06 May 2026 03:12:58 GMT openai-organization: - user-r3e61fpak04cbaokp5buoae4 openai-processing-ms: - - '276' + - '512' openai-project: - proj_2Y4H7T2XPShczTQx5fqMXdNp openai-version: @@ -117,17 +107,13 @@ interactions: server: - cloudflare set-cookie: - - __cf_bm=D0JLG2055c9_Qp1pj16jx7tyQVtYPJ5Jlr8srrADKis-1778032439-1.0.1.1-wSFVNPO_rARMYH4a6F.V29bDImlV73C.ndtzHFwdmMoE0v5MVnLwn7oHEao8S1uWbmzJhYfZSGRIKmWJzogh1mZ0.OmOvQwzRf7q5oEMYc8; - path=/; expires=Wed, 06-May-26 02:23:59 GMT; domain=.api.openai.com; HttpOnly; - Secure; SameSite=None - - _cfuvid=vNa5HJF6CAbxVrsu4xC9Rqxc03APkSzlo2KtIijWnHQ-1778032439396-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + - __cf_bm=Xlm1i2d4hNcY9FYDrgbksx5Ak.InLICBWuWA7k.o5eU-1778037177.9137826-1.0.1.1-GD76qNB2IKnNrhlsjKWgXNF6BpybgPQmNoOTYCMh4ejIhEDWMMph.l_WZvek4hrh5qeLqHfJ28EH5H2c10KBdqGhET8jLthIUOa15fXX7oxYNG_MZ.CHZ9Fuj4bT9GU.; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 03:42:58 GMT strict-transport-security: - max-age=31536000; includeSubDomains; preload transfer-encoding: - chunked - vary: - - Accept-Encoding x-content-type-options: - nosniff x-ratelimit-limit-requests: @@ -143,7 +129,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_f37a6e34cdac4945976e57df6a3ca8fd + - req_445d87d531af499daeb09f7826886b8c status: code: 200 message: OK diff --git a/tests/cassettes/test_openai_responses/test_responses_round_trips_encrypted_reasoning.yaml b/tests/cassettes/test_openai_responses/test_responses_round_trips_encrypted_reasoning.yaml new file mode 100644 index 000000000..023c034f1 --- /dev/null +++ b/tests/cassettes/test_openai_responses/test_responses_round_trips_encrypted_reasoning.yaml @@ -0,0 +1,356 @@ +interactions: +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Pick + a clever country name, look up its population, then check whether it can have + dragons. Be brief."}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"lookup_population","description":"Returns + the current population of the specified fictional country.","parameters":{"properties":{"country":{"type":"string"}},"required":["country"],"type":"object"}},{"type":"function","name":"can_have_dragons","description":"Returns + True if the specified population can have dragons.","parameters":{"properties":{"population":{"type":"integer"}},"required":["population"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '722' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/71XWZOiyhJ+n1/R4fNwApD1vLmhICIqrmcmiAIKRLZiKRUm+r9fwG61e3oi7hJx + fTAks3Kp/NL8kl/fXl46vtP5+6WTwRyZpMsIpGjRPMORXZZySIoTXWDRFO9YXUagRM4SBbLbpSyW + Z1yGtbnO98ZFYp2gXby7SeIc3uR2BkEBHRM0Oorna1ue5qlWlxegwHljYycRCmF97mZkATvwsgTH + TV4uCHN4E/th6MdeLftVP9YCBEqYNfYOPMMwQfVDrXi9BX53+Sk002phliWNZYzDsBW4GUwxjO3S + RDAGYVHWSvIvstX58bsz04EF8MP82dKP8yLDduHXl36WR+BqJrhAuDCLJIC/K4skCU0bhB/dRYkD + w+ZOHioI9i+WoEmaI0iGoLud+4EMNOGezW6Rask/bWluBXpAm/8ZWEHkHbsBVuRIKHAC6Vo8DQTh + hkXrpCgRvEEL8iRuILir6pJlJWrKbCdxAeO2Bbxe8+kjIpOBBrltEBuzWB8YW06R9+t5ni0sbX02 + gpEZbcg1QKP1hdFJlkulrrgBVzQ4GWMJDa04PRwZbFu5gayRvL86Z4IgWG8ZBZUHT0LXYUdld1EE + qrQnSo4ieR0rZ6wOh/mih1fq3jP8rjII0zxBWlAxhaAMVuVAFzyDG9mZWAWrfDTTJolSTg8CvZwY + ispr5arAqevK/mouKT6xEbeStcgnV2QtSWuUrOH4pKygI3CcaOpbKVIFo/R6R0nDVzI7L3ejSCMT + cb2wgZkJfkokRmH39+RMt/gySgvqgCfunqeWWo514miJs3nXlLtyts1ExdDYfLmQVYK9+Ny+HCn0 + DBnVGYPteoO7YD4OxghWrBsEG2dJc0FxusrustiQZ6WSNgItCodetNEjOln23TzzTAvtmaWo2Zy6 + mSJigLfm8mrsaLHf3RCTqZoMp9HZj7dLyByG4s53Np7pLqI9POdaJqiL7kTDO1kdokWyE2Lblvub + FR6a9my4WJUWkwuUvhghJw4de7qiMNCwtpH2/aB/1aeYy6mxxpKJdLog3fLMgKTigb5zLrsZmSpL + ja+jKMZQVK14zc7OkDxq+lwxB2Sw5emUkg/eKZ76cRisnD2pDFfkvqf0qjklnibqfOcUVUAX/Q07 + 73Nj06MDow8inouW2mYwG1aD3Yq4CvZoA9lpNOyFmryohHBb9WVH2gUkWfbpmW1soozmablUgqWm + pjTAWJLmYbDexfR6xubdPvLn+X4Es52Ap5fJsOfLe9mE3rXq2ds5F7NuNZISdLACZ76YHqzrwoWX + /gEcJ0MmWJ8YYcLBKEkm5algRHarbk0C9wqXmCribNdjexsc9qsNbywoiSt3l9UIYEtaoQpNQaFH + PdHr9tM4To18wkCR7LOKTKuXPOPZ9TCaKWhywaPpXBHOVSz6UARzW3Tko3GaHt3JKPFywvL7Vjo1 + lHhZDbB2PBxT54DFrehmg+mxIhmtOqPo4DuiOMp54cRN5olzFgZ6PAvKmDXjA9EbTqTzAfqAzRNp + oxKMsL+sp0XFBCk3JSR5jC7d5YUcRr1uFYKRM+aOy3kGrEDXS0/qn2bEXPZFchGd8DUsNoeejPJq + e1pn6Ulxk5gliMw38DwS0xOp48swJR01NQp/tIl5wmGAkgnmVFxkzhipO3xSK10NUDaGe2kMpGPB + cpQ2WZbcmgtjIXaP3bMvkAfE296unpj97nAUT2RtRB/Y6JypYhBg+xJoFwsqRDqX9cE+jHS+2oVD + 0Ys4czO7hhREU1Zbn8qBHYmz8XhGq7RCQdeezi6LvqtpHEJHTE3plXdxPVJCepTq3o4I+2eKTmfO + cjCbBnNjT/bUuSyfWE0Xd9UBu+yWG85PScmp14Ww6suDbjq8rIEgY1afCqMg8LRgi6Gh6BUy9scT + WKZ4b/aFx6jNcRSBrCGmf362stfvX8161/7zrAddxiVbEqcYm7YcngI0bTsc+H3WuzhuOa1lqKck + /sDZrRJkHo5qDmj1v37UR3BcZOWPzt8/OnrN5kkGfnReH+cb1+Yt6/YnLvnC1TZJIGu8dpIkLuDn + hVo+hY9B1GYXJkmAkYkShMMbF94qUn//bPkQgaz2CMOPLFsT9W2VQPWWUlMX/ILta9XZT3Buvi8y + twzvTIuy+uJF7dI+QjOA5R91GWz48MbTHZo53uj7waH3NQa6bpK1xHn0vePbZZ/gbtzf95ocuLAo + 65wa364PP2wxOczOfn2rwn/fi1yAw6Lztm4lGfywUBUwQs0ugVs59VaAAl6LR3J1ahF4PD91SHvu + uRE7Z5hZSe635exE0PFx9FjIbkAckzq/Rg1wkXTuivz3zeVzIz6awIG5nfnovbRLWF8gzl+KI3yx + cZbVhXl5NMZL4raaHEG7qZfz4vqtPxC+vPXnX/9Of92PNJ0V1dBm+VNRbtjXxawr/1HerqJtmE/i + pyvWC2TTEE/K1/vv1++PEM2O6mfQuRfrU4C77OeT0XuQtw39SQMcx79VQn/Ove2Pb5+itzm2C37z + H/py/PzHgBm1pxf/MzxP0NkgfjmCM3xxMuDVf8UvgKqPmM0R8+3I/4LTE9p/hMqvl1zv9p7x32H1 + eWT9/+C6z8YiQWaYeHUhrMYBeReidgyKwk2Q1Ti+V6Pj+DmwwvcXNJwDDz5mhB9/eMsRme+/y59e + nX495n89J52HIflhmnx+eRLYrxRf+b3P2Icx1/3gu0gKED60FC/e5xTOP87UuoeAAwrQBHj99vov + v/3TYzEPAAA= + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f74d0b45806549d-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 03:14:34 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '2715' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=JaAjlN.SsTHssMfEA38GX9HitAbY0eJ2RFPOYXHE.dM-1778037271.7367537-1.0.1.1-ITxNEO9g5Dnpe0xysdsQx6Pb4UfPCLUH47.8gv2jcppI9YSzDXqxNV1wVH1S0DOjPhkjvEoameT18PZ3jubWSiQeRsWVcFTBf8rKD.riIQ5F_txQnQ76olVhtIu0ZumI; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 03:44:34 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_83a358e390da43f0af03fb3186d6e121 + status: + code: 200 + message: OK +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Pick + a clever country name, look up its population, then check whether it can have + dragons. Be brief."},{"type":"reasoning","id":"rs_0f4809b27460351d0169fab21897dc8196960e8680fb72a88d","encrypted_content":"gAAAAABp-rIaNe6WknTMnPCTW6JIYUOsrQbNUvTkE_mV0UapEUw4P056qF39VaxpCjTGFpDbnqZh4ucbsTpbEIYxdv---5gRmkzgej83d5Ey3QtkLFY-y6107PuJvuLDDsQAuSLYgTi3JClqsopNkz4t8JCSyCP8gT6Ecr9zkSsEMNHoJyKZ82RHTJL7NyStuqffIiSOFJi-V9WFbQsHxpbR0bEoUeGjJSed8669_PWFmL8TygAhFNux0rvRXEmN0o9UQca_r8iq-oTtcBY0MPb7ymqt1ZuHfY71RNsuP-hb9MO3_I3IrWr9JTN5sRQIL-5wi6YyEJ2MpTzvuaWUVu3aOGkGpez5fkkVdR26ktjxIfRtV0vJzFV8298ZAmVPm2oRBfsrg_bpY4R9Nc6LVKp-CuW_RxTX29B3V-HKLoDKmvinWRe4ZD9XidVg_fQmYevsNr8LQ3HNuXILDpQoX8nccIBVSuD_cMDQSyb4s81PQEpdnldcKS1uaNuNVFYBkBxPKu6s1GN50oFjwpPbg_k01nCPXdwXM0qJRN7evsJTD9LbnU5Mve0hNPOJ_C0kW72q1IZgjnKinlkSdY0JDS0YAJAzO19jHLOXdtzk2tBV5OB6G_g2kTBam76mRNVCMDzCXS-x8cEVe5KmDAlNIQz8lWzBIdFXk00yB2McTVmr272IyJkRNLq2auuFFOlkUXn2UM5s3BpiOsYEerX8uKwHDAiIYI_egxzAcWO6n5fzEFopZbkdOQKZbxQfewBZahHD4kUj48H6emooHyjt495WLW_-uAtf-KJ9MXA5AVulBzV7TQ1F6yXwSEaubFSpzpKatPmA9g3BqnnqTsH4e90B5JI2Lwsr75UDmMJpHwuEKOJ8vzn9ie9aOc9dIhTjKhfHEogs-biBbqKTJnRzCuNhZhqdZu9W9frCKhz04NzvpmZid99Es78j6HOodv8CPnMkyn5_nZ-ADHFvZeia5soFVL-48YwUKtz4kq6K-FIGpw3Rw0DmA3zlaEdG6hROrabkPPygFBjM-OIi90QmjuxltVZAIpszWjUrqjJfon5--riTuOm9qj0PuwDq0dLqTtiEVn7-d4aJr8_K9QrdGpLXujLzPLkprGeYFGaFht561NHRy6U6ln8nfh3vi80Zp7cgX4-2B3DEnHINE2Z5mvrL9kkucwkNwbeJ-qOIPCYlmP7zXlD9gm6_VMxl1epK5NUjyCcm9MGGM2L2J1efcKMwQBfNN6pphu1K2Sgwfg0FpPmqPgX-lBv12qMdRCMKkOTY0ALOIIj5NP9XzZuf5W6DOjoy6LxQ8SBIC3qDwUa8Iu5PK8EkkgNkWueTJPzpTYhjaRquY_B8","summary":[]},{"type":"function_call","call_id":"call_uy7tfNVokIN7NjFF6k7OtLyl","name":"lookup_population","arguments":"{\"country\": + \"Pundora\"}"},{"type":"function_call_output","call_id":"call_uy7tfNVokIN7NjFF6k7OtLyl","output":"123124"}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"lookup_population","description":"Returns + the current population of the specified fictional country.","parameters":{"properties":{"country":{"type":"string"}},"required":["country"],"type":"object"}},{"type":"function","name":"can_have_dragons","description":"Returns + True if the specified population can have dragons.","parameters":{"properties":{"population":{"type":"integer"}},"required":["population"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '2423' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/71VS4/jNgy+z68wfJ4Z2M7Te+t5L9ttgW7bXRiyRTuayJKqR7rBIP+9lBS/MhOg + 2AK9BDE/8SPFj6ReH5IkZTT9kKQajKqydr3PyrrYrbfZapPTLN+WLamLnLTbItvn5bZe53S/o7SE + ksKmLdNHTyHrF2jsQCOFgWhvNBALtCIey3e7fbbaIXnAjCXWGe/TyF5xwHPRqSbNsdPSCZ9XS7iB + aGacM9Gh7RU/0aDIGbT3p3ACLhV+IHCJgQfKm9CbgILW0nsKx3kwtBr+ciCac6VAEG7PCGbPWcCY + GMgqCpYwbuaeTBirXWMZXnpu78n3SjqrnK2sPMJb0ErJq4bwJV0vKXB/p07Zp83z5qnIiu1Ttn4q + Vul4QBMfbu4WI6Hlz1CaWKBR2ra5L2zdwH4fhM3qVVm3BdB8vVuvorCBxJ4VBBonwj1D1hN8T8cA + Et25HoQN+OvXVEnleEj/a/ohL1Z5sb5Mpz1xFXMOf1/+/n1/5Py3n4yQv3yxL1/++Hnz8fOWTR6C + 9BCPi+pATiiRJp2XIhy44O+3UCBFNBICX5YdlYu9pbBtUX54R36ETkw6Uw2dHRMcS6803toiZXOA + 6gjnu5jGyoircGmxPkQ9cUCMFIu+hraVOgzTgXWH611T4/qe6IF+bHRDWrBnzMlztwwWbW1Anxje + yrJhUFriuE2v8yc1LCbMQq98c7lgz68FsPDdTslhaj2ZvmftEc7Fsl9TPoGupWGhnGkPlLl+mtAo + xEFifh4mzsp0BMzbVr7twqkHKJhGMzWU9jPgBYRJ7AGSxmmNhUmmtktkGxCjoPH1oknLAh/hSYM7 + x+rz89v24lIenaommumI76wepdVmVpSoPRYTK7+0h90UwtyYZ1fEjeIbYgZexv+XxymEX1pMAx2L + dRNgtH2bOQ1Brit7hhBKWazEp3nuoT8ebqKHHMPG9zM0V/3HBfsVmRJ2K89MOhzyxA95ch3y53+x + B/6DTjO170rFhIUuPjw/ptUsyP8s17gbrVQVlx0WovYE2WhUYQ2W+2jQqONQjZQyQ2o+vNjOkA6m + HcHE4tnLy+LxLTB7TF+n/Y+Lkk6e2WKd3D6nRf4e8B7vuGTvUVtpCZ8zr8Y95cxyp2IPEUos8fyX + h8s/3J7RqkIJAAA= + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f74d0c81c4d2b4b-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 03:14:36 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '1077' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=Ff7ZRSpcLAJPDQ.asF9RF4bqWtg97VpC0E51yPz81.U-1778037274.8993971-1.0.1.1-lFBHWH1sK41g3Of_MJ292u9sow8m9djH6vBQSla3pNCvWDtdVwVDdW.ko4QJQaLNA6RhV21md8copjjSWmsVnZJzyKsQgS.IPimoDzj3d9Lt6eQuV2eXHZenOh25fKxo; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 03:44:36 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_477366fddd32402daba00618a4c16efd + status: + code: 200 + message: OK +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Pick + a clever country name, look up its population, then check whether it can have + dragons. Be brief."},{"type":"reasoning","id":"rs_0f4809b27460351d0169fab21897dc8196960e8680fb72a88d","encrypted_content":"gAAAAABp-rIaNe6WknTMnPCTW6JIYUOsrQbNUvTkE_mV0UapEUw4P056qF39VaxpCjTGFpDbnqZh4ucbsTpbEIYxdv---5gRmkzgej83d5Ey3QtkLFY-y6107PuJvuLDDsQAuSLYgTi3JClqsopNkz4t8JCSyCP8gT6Ecr9zkSsEMNHoJyKZ82RHTJL7NyStuqffIiSOFJi-V9WFbQsHxpbR0bEoUeGjJSed8669_PWFmL8TygAhFNux0rvRXEmN0o9UQca_r8iq-oTtcBY0MPb7ymqt1ZuHfY71RNsuP-hb9MO3_I3IrWr9JTN5sRQIL-5wi6YyEJ2MpTzvuaWUVu3aOGkGpez5fkkVdR26ktjxIfRtV0vJzFV8298ZAmVPm2oRBfsrg_bpY4R9Nc6LVKp-CuW_RxTX29B3V-HKLoDKmvinWRe4ZD9XidVg_fQmYevsNr8LQ3HNuXILDpQoX8nccIBVSuD_cMDQSyb4s81PQEpdnldcKS1uaNuNVFYBkBxPKu6s1GN50oFjwpPbg_k01nCPXdwXM0qJRN7evsJTD9LbnU5Mve0hNPOJ_C0kW72q1IZgjnKinlkSdY0JDS0YAJAzO19jHLOXdtzk2tBV5OB6G_g2kTBam76mRNVCMDzCXS-x8cEVe5KmDAlNIQz8lWzBIdFXk00yB2McTVmr272IyJkRNLq2auuFFOlkUXn2UM5s3BpiOsYEerX8uKwHDAiIYI_egxzAcWO6n5fzEFopZbkdOQKZbxQfewBZahHD4kUj48H6emooHyjt495WLW_-uAtf-KJ9MXA5AVulBzV7TQ1F6yXwSEaubFSpzpKatPmA9g3BqnnqTsH4e90B5JI2Lwsr75UDmMJpHwuEKOJ8vzn9ie9aOc9dIhTjKhfHEogs-biBbqKTJnRzCuNhZhqdZu9W9frCKhz04NzvpmZid99Es78j6HOodv8CPnMkyn5_nZ-ADHFvZeia5soFVL-48YwUKtz4kq6K-FIGpw3Rw0DmA3zlaEdG6hROrabkPPygFBjM-OIi90QmjuxltVZAIpszWjUrqjJfon5--riTuOm9qj0PuwDq0dLqTtiEVn7-d4aJr8_K9QrdGpLXujLzPLkprGeYFGaFht561NHRy6U6ln8nfh3vi80Zp7cgX4-2B3DEnHINE2Z5mvrL9kkucwkNwbeJ-qOIPCYlmP7zXlD9gm6_VMxl1epK5NUjyCcm9MGGM2L2J1efcKMwQBfNN6pphu1K2Sgwfg0FpPmqPgX-lBv12qMdRCMKkOTY0ALOIIj5NP9XzZuf5W6DOjoy6LxQ8SBIC3qDwUa8Iu5PK8EkkgNkWueTJPzpTYhjaRquY_B8","summary":[]},{"type":"function_call","call_id":"call_uy7tfNVokIN7NjFF6k7OtLyl","name":"lookup_population","arguments":"{\"country\": + \"Pundora\"}"},{"type":"function_call_output","call_id":"call_uy7tfNVokIN7NjFF6k7OtLyl","output":"123124"},{"type":"function_call","call_id":"call_jwY8kllWAsnoSXtjXZQ5KR6i","name":"can_have_dragons","arguments":"{\"population\": + 123124}"},{"type":"function_call_output","call_id":"call_jwY8kllWAsnoSXtjXZQ5KR6i","output":"true"}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"lookup_population","description":"Returns + the current population of the specified fictional country.","parameters":{"properties":{"country":{"type":"string"}},"required":["country"],"type":"object"}},{"type":"function","name":"can_have_dragons","description":"Returns + True if the specified population can have dragons.","parameters":{"properties":{"population":{"type":"integer"}},"required":["population"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '2645' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/71Wy47bOBC8z1cQOs8MJFm27P2KINhbNhAosWVzhyIZPpwYgf89TcqSKHtmDwmw + N6uLrCa7qpv++URIxln2F8kMWN3kfbXPD21ZV7t8sy1YXuwOPW3LoisZq/bFYXfYtvuS9mULXdHW + fZ49BwrV/gudm2iUtDDGOwPUAWtowIq63uebuqx3EbOOOm/Dnk4NWgCuGze1tHs7GuVlOFdPhYUx + zIXg8oixn/iJAU0vYMJ+BmcQSuMHAtcx8UR5l7qOKBijwk7phYiB3sA3D7K7NBokFe6CYP6aR4zL + iaxh4CgXNt3JpXXGd47jpdP4QH80yjvtXePUGzyCTinRdFSs6QbFQIQ7HbV72b5uX8q83L3k1Uu5 + yeYFhoZ06bYxE0a+xNKMBZqlHezxP5Rl9SYPyrZ1Ve9hAzsGHc1bGvNFFnfREHnAWnqEBfhIwgh2 + SjqQy6HSg61opzLBDzfvjguolMrRqbRfvq5AoY7aqPYdJBIh7yd0kDKUnKgllGilvYhkRPWkKDfP + RVmRf3yZFxWhkpEL2GfCHemoxC1nIMzQI6Z+zWby6+3XnC/TSB4v0XP0TUOl/Y42nGGjRESptRyL + hdV4mngiB3rYoAVArO2Ajho9r7Gd0Jbwji0ROnPlbTN1XBPFni2BxRm0Q8ruBM0bXD7EDASZRkNl + ZXUafYaNa5Vc9Rv0vTKxsid+PN0umVk/DNRM9HMDWtqDu+CZAnfPYdVuFsyZ460cnxq4p16M6qOp + lIFV5zsYdDC9j/HiVoCbzLfD4dEGunwn9orrxrLfjnwG0yrLYznR1Iz7YZkcoxAnheeLynmnshmw + jy02pem9jFNgEZ+B7QzXU2k/A15AWuJOQDpvDBbmzpQBsRq6UC9Geh75qCAdzkJnLq8LtaRDTCqU + evO6WWiWJcFZA0prbFKUUXssJlZ+HY8NG9PchZMr4qQLhkjA69IaSwtmYZhyA2zV+kmCOZb07TIN + xqckQShjfKzEp/Ts0R9Pd9njGeNLFHooVf33BfsbmQi/lyeR7nFiPAiFS5qwpLkt+ROdErU/lIrj + 7D2OD+LvaZUk+Z/lmmejU7pJpnw+B3Ucg4f9GDCo41SNjHFLWzH9k/DxuZpnBJer57gs6+dHIHnk + Z8vEQcmWnflqnNw/82X1HvAe7zxkP6J2+P6JhHlbzHPK2/VMRQ9RRh0N/Nen6y8ML4eS2gkAAA== + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f74d0cfcf8b2b4b-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 03:14:37 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '1306' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=QSHgROUcoFXh6zyyvGCnOLGXX2c4b7S8nJowB4RJhtI-1778037276.1270735-1.0.1.1-m3I6gMTwPPLlpeHgduEBRKmVIQGDDZcoItTTaVkwIUQXMOTcCG5_fl8MQIwjyawnDJPGxUthuGFfjTpsPX1OSBOXGmAUziyOhJ04qWwyUzM7fMtNEWtTmOgvh4au55D0; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 03:44:37 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_e20848e56750417e9dd0940f7c4fc82e + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_openai_responses/test_responses_tool_use.yaml b/tests/cassettes/test_openai_responses/test_responses_tool_use.yaml index ce9086a00..4cd3af59b 100644 --- a/tests/cassettes/test_openai_responses/test_responses_tool_use.yaml +++ b/tests/cassettes/test_openai_responses/test_responses_tool_use.yaml @@ -41,37 +41,37 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAA/5VVyW7bMBC9+ysMnu1Am2Mr5x6KJi2KopeiKQRKGjmMKZHl4sQw/O8dUtbmOAHq - g2HO4zzO8mZ8nM3nhJXkbk4UaJkF6RroJgUooEjW6SYIb9OKplUCQRUWm5CGaR7ROEiSchVBkAcB - WTgKkT9DYToa0Who7YUCaqDMqMPC9XoTxFFyG3lMG2qsdj6FqCUHvNc65bTYbZWwjYurolxDa2ac - s2aLtiMe0SDpAZTzL2EPXEg8IHBqH+4oL56OPQpKCefZWM69oVLw10JTHDIJDeXmgGBwE3iMNR1Z - VoKhjOuxJ2u0UbYwDJMe22v6mglrpDWZETt4CxoheFZQPqWrRQnc5bSVZrm6WS2jILpdBskyikl/ - QVH33NitfQktv31p2gL1ra2KDxpbFmHiG7uOoiqENF7lMazWiX/Ok5iDBE9jG5+nj3qA3+ujB6na - 2hoa4/HjI6GP5C6M4nDxSHL8GcVxeBpuO+Ksjdn/fL7/vHuovlTpge/yX/uH/P6H/PTtZzN4NLT2 - sdWWGyb5gXjghN9/fGEkVUgEfFpu7FirKYlyxbbDlbYjtGfC6qxTdBtYX3KpMFuDlMUTZDs4vIsp - rEhzbhiJkqe2jzgYWjQTPUNVCeWHiIuXc4pE27qmqmPv9a1pBeaAITnqisFEzRrUnmFShnXzUVGs - DzmPnVAwGSwDtXSast4envM38GqG2DCymg7nkSr8vbbq55D3oHKhma8mqaFkth4Gs+3Dk8D4HEyt - EaQH9FsFX4pvaH0JulBMdpX9elbA3LwILEWdg9I3HwilR5xEauyR0qP02iZiWbCGU7sT9YVhFCZr - DGzbRdR9Touxb/5/vrMrLMQtK6ag7KvVx3XxVH/6M/Lunjvv7BFCy5K5clL+fZy7V8rsIgzUkWJ+ - 5bthupg6I2TGxRYrmDuCoDdKP2DppjUo7Gm3ykjJNM159x9gNd3CID/WTBbparN4ax9t5+OwUHAC - y8ExmAj1cj9H8TXgGm8/ve9RG2EoH8BN2A+A1dNhRenRkhqvqdPs9A/gX36dkgcAAA== + H4sIAAAAAAAA/5VVS0/bQBC+8yuQz4BsB8cO115aaKVKPaGCrLF3HBbW3u0+gAjlv3d2Hb8CVGoO + UXa+nW/n8c3k7eT0NOIsujqNNBpVxlkdJyxjNVaX66TO4mS9aaBKqirDOi6SzbrKcVU0RRrnAJgy + Fp15Clk9Ym0HGtkZ7O21RrDISvBYkudFvMqTfBMwY8E6431q2SqBdK93qqB+2mrpOh9XA8Jgb+ZC + 8G5Ltjc6kkHBDrX3Z/iMQio6ELDvHx4ol08XcUBRa+k9OydEMDQa/zjs6l2psANhdwTGF/1l3g1k + JUMLXJi5J++M1a62nJKe21t4LaWzytnSyid8D1opRVmDWNK1kqHwOW2VPc8usvM0Ttfn8eV5uorG + Cxr8c3O3/iWy/A6l6Qs0trapP29svYY0NBaKJt9keZauWYLNZhOeCyR2pzDQuC7kGaKe4M/6GEDQ + W9diZwP+dhfBXXSVpKvk7C6q6Ge6WiX76bYnLvuYw8/uuv1yk/+6xhU8sttbJb5VX+Ob7ffJo4M2 + xNY6YbkSuygAe/q+D4VRoIkIxbLc1LFeU4rkSm3HD9pO0DOXzpSDovvAxpIrTdlaoqwfsHzC3aeY + pop0h4ZF6eVD30caDCO7hZ6xaaQOQyTkyyHFyLi2BT2wj/o20KDdUUieuuG4ULNB/cwpKcuH+WiA + 6hMdxk5qXAyWxVZ5TblgTw75W3y1U2wUWQvTeaaKcK+v+iHkZ9SVNDxUM2qRcddOg9n34UFSfB4G + Z2U0Aua9go/FN7Weoak1V0NlfxwUcGpfJJWirVCbi38IZUS8RFrqkTaz9PomUlmohku7F/WRYRYm + 7yxu+0U0fPZnc9/q/3xPPmCJ/LLiGtlYrTGuo6fG0/3Me3jusLNnCDDGfTlB/JznHpRychQG6Ujz + sPL9MB1NnZWqFHJLFaw8QTwaVRiwTdEbNPV0WGUR4wYqMfwHOANbnOTHu8UizYqz9/bZdn6bFgpN + IJsc44VQj/dzuvoI+Ih3nN7PqK20ICawSMYBcGY5rCQ9YGCDpvYn+79G2nD3kgcAAA== headers: alt-svc: - h3=":443"; ma=86400 cf-cache-status: - DYNAMIC cf-ray: - - 9f745b477f340044-ORD + - 9f74ce7299e4cb75-DFW content-encoding: - gzip content-type: - application/json date: - - Wed, 06 May 2026 01:54:23 GMT + - Wed, 06 May 2026 03:13:00 GMT openai-organization: - user-r3e61fpak04cbaokp5buoae4 openai-processing-ms: - - '1084' + - '1326' openai-project: - proj_2Y4H7T2XPShczTQx5fqMXdNp openai-version: @@ -79,9 +79,9 @@ interactions: server: - cloudflare set-cookie: - - __cf_bm=z7uNE1DqeNcWOXy7RIEwbyUEWlzdoMxhY9qUkpfgq8E-1778032461.9979239-1.0.1.1-VqVdBmRJortunHk6vz6JIxYW1JMEnm17BsExYD9AGOJBsRDJIWU_K7O0Hnxp_tK6ziVKx4HG7qrTcXV07bZCPdoED.Rg_hKX.CjJv529BuAWX_aerfhFo8Xob0p8.Sfs; + - __cf_bm=zXi8aTY7cDmdwne_.I.3Hf.KrkQjzxMBHFbO2WWZqFg-1778037179.2921715-1.0.1.1-l3RZkU_etG9t6aSFPSTnlBiIypr5vSKjMdMV9iEuWrDk59KcVyvnaNk5TDFILQjTF1bFbvNJ5NmInvZIg_tp2BdSmbNPLyyqdw3RXJ9W9MUEfl9QCXSFs4E6vlaRqupr; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 - 02:24:23 GMT + 03:43:00 GMT strict-transport-security: - max-age=31536000; includeSubDomains; preload transfer-encoding: @@ -101,14 +101,14 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_879af2c359a94b25bca83aca9ef2e65f + - req_e6f3256822534dd38be1cf139234e596 status: code: 200 message: OK - request: body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"What - is 1231 * 2331? Use the multiply tool."},{"type":"function_call","call_id":"call_jKHkLfJf9ylkbYvLbKRpDNTn","name":"multiply","arguments":"{\"a\": - 1231, \"b\": 2331}"},{"type":"function_call_output","call_id":"call_jKHkLfJf9ylkbYvLbKRpDNTn","output":"2869461"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":false,"tools":[{"type":"function","name":"multiply","description":"Multiply + is 1231 * 2331? Use the multiply tool."},{"type":"function_call","call_id":"call_nJmCK7SJe3ajdYYplIbH0KgL","name":"multiply","arguments":"{\"a\": + 1231, \"b\": 2331}"},{"type":"function_call_output","call_id":"call_nJmCK7SJe3ajdYYplIbH0KgL","output":"2869461"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":false,"tools":[{"type":"function","name":"multiply","description":"Multiply two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}]}' headers: Accept: @@ -148,38 +148,38 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAA/5VVSY7bMBC8+xWGjoY90OZFAfKEALlPAoESWzYzFMlw8Ywx8N/TIq3NYx9yk7rI - Yi9V5OdiuYwYjb4tIw1GlXES76o8JYe6qYDmWZzsioYUTd7ku/xwSIqsSiAhCdD9NsuKLMujdUch - qz9Q255GCgMhXmsgFmhJOizZ7w9xlua7zGPGEutMt6eWreKA68KmitRvRy2d6PJqCDcQwoxzJo4Y - +8RfDChyAd3tp3AGLhX+IHANB/eUd0fnHgWtZbdTOM59oNHw14GoL6UCQbi9IBi/xB5joicrKVjC - uJnuZMJY7WrLsOhpvCUfpXRWOVta+QZfQSslL2vC53StpMC7mo7KbrYv200ap7tNnG/SLBoWaNId - N90WTsLIq29NaNAw2tYcn052G+d06ydbHHZF2iT5HpoiL2jhz/Ms9qLA84Ax5Agj8GyEHqylsCDG - pKaJzWj7NsGHHXb7BUQIaUnf2tffM5DLo9KyeoB4IuRN0ixZ/nJxTPfLNMPv78vVKl1jmet8l6xW - 0bDpevsaeCJ1IsYn1zDUQ0mEeUd5DbCW3KPEGIZNwCoXPY/nQG1qHC3w+ZhRKUHLCm2CcoMHckPo - zKQzZe+k0g9xGDUW3SqLlPUJyje4PMU0dO0PQonS/BT0g4Y0Usx8BE0jte8Yl++3GiPj2pbonn3w - lSEN2Aum1FE3DGYuMqDPDIuyrPdlQxwPQ0WtSA0zQ1toVadl5+PJrf7b9G65YWYtGf8nqvHrQtdv - KZ9BV9Iw303UKmWuHS+EMIeTxPz84JyV0QCYr87pj2mc8OYeZ0/B1JqpvrM/sEKm+GVp3yW2oq1A - m5dxtSBt8M5t2Yh0EmlxRtpMygtDxLZgD+fxzg93gUmaDL12DBfgIOqZKar/27t4wBJ1lyTTQGeW - 9nndHTX8TYw52j28FROEUMq6dhL+c1q7V8riLg3UkWb+qenMdOc6K1U5uRfiIai8wYpDCGicaX+F - RpQZUvH+7XH+ghvkx8TsAi/y9df45FUY1OMdSMeN8Uyo9+9CcngEPOId3PuM2uKFySfMSTo4wJm5 - W1F7hBLrRXVdXP8BMqG3+wsIAAA= + H4sIAAAAAAAA/5VVTY/aMBC98ytQjhWskgAhVOpPqNT7qoomyQTcdWzXdthFK/57x3YSEhYOPUFm + PM/z8d74c7FcRqyOvi8jjUYV8b7aplBleZlWh+0miZPs0ECZlBVmeZwnsMkPmMEO4sMW8jrLMFo5 + CFn+wcoOMFKY3l5pBIt1Ac6X7Pd5vNkneex9xoLtjIupZKs40rkQVEL1dtSyEy6vBrjBYGacM3Ek + 2yd9kkHBBbWLr/GMXCr6IMc1XDxA3l2deC9qLV2k6Dj3hkbj3w5FdSkUCuD2Qs74JeTJxABW1GiB + cTONZMJY3VWWUdFTewsfheys6mxh5Rt+dVopeVEBn8O1skbuajoqu9697NZpnGbreLtON9F4QIO7 + bhoWbiLLq29NaNA42tYcn0+2dhY3WdjRSHebJq92+30FYYIexV4Uehw0Bo4Tx7MRemclhUVxS2qa + 2Ax2aBN+2DHaHwAhpIWhta+/Z04uj0rL8oHHAxFukm6S5bdluqGfH8t0lWeH1TZLovHstf83hkfq + BMbn1DCiQQHCvBOrRreW3HvBGEa1U3GLAcdjECU1TRT5fLpEkEBhReogluEDlpHrzGRnikFAhZ/d + OGGqtVWWIKsTFm94eerT6Loe+BGl21OgDenQSDGTDzaN1L5RXL73NUama1vQA/ooJwMN2gul5KAb + hjPxGNRnRkVZNsixgY6HWRJFpMaZji22ylG48/akr78fWp8bZdbC7XtCFn8udL1P+Yy6lIb5bhJF + a9a1tz0Q5nCSlJ8fXGdlNDrMV8EM1zSd8Jq+zb5GU2mmhs7+pAqZ4pelfZfUirZEbV5upwW0QTL9 + sZvHUaSlGWkzKS8MkdpCPZzbnQzuDJM0GUnsGPbeSOqZFsr/i108QIncbmQa65mSfV53V41fEz3e + VB6eiIkH6pq5dgL/Na3dM2VxlwbxSDP/wjgx3anOSlVM1kE8GpUX2CEPBk0zHTZnVDMDJR+enM7v + tZF+TMz29mG7+mqfPAYje7wC61tgPCPq/XOQ7B85HuGO6n0GbWlP8glykowK6MxcrcQ9qMF6Ul0X + 1381ZAUOAggAAA== headers: alt-svc: - h3=":443"; ma=86400 cf-cache-status: - DYNAMIC cf-ray: - - 9f745b4f2f7d0044-ORD + - 9f74ce7c3b71cb75-DFW content-encoding: - gzip content-type: - application/json date: - - Wed, 06 May 2026 01:54:24 GMT + - Wed, 06 May 2026 03:13:02 GMT openai-organization: - user-r3e61fpak04cbaokp5buoae4 openai-processing-ms: - - '1355' + - '1106' openai-project: - proj_2Y4H7T2XPShczTQx5fqMXdNp openai-version: @@ -187,9 +187,9 @@ interactions: server: - cloudflare set-cookie: - - __cf_bm=n0X3ttAhrLg1o4.WV.ZnmBJk2WZojTqaq1DsDoffi5w-1778032463.221904-1.0.1.1-vn34lp4AgkusR7VdGs7M8S_Gq2eOxBEbOdidg2ZEMx3iopML7ykmJupC_aAHReNHK8vo8.IXAw0zdXeZIhClirFAZoqB1sglnuvDE.x1LcjUY6Be2mdr7Aw67UkJG7Es; + - __cf_bm=yqi2SzRPEQIj_22_ctCcbWfMKVpSPEKe83ZGD3EBaLY-1778037180.8329482-1.0.1.1-enl4RmONGIjXc08ZO75dW7K7g.J5p7W_GNXjJSYbhPlgeJK0.2h4kJfDALyQFDJUpAFXFOhfg.OC3VA.I7QfOELuUPnL1HR2iM7vf7Giaqal.KxFdQJDyFBy5H6GRQcu; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 - 02:24:24 GMT + 03:43:02 GMT strict-transport-security: - max-age=31536000; includeSubDomains; preload transfer-encoding: @@ -209,7 +209,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_552a72b066df401a8c127fdb2afe042f + - req_3a3a8ea3224a44ef8a1ccfa281e21c3a status: code: 200 message: OK diff --git a/tests/cassettes/test_openai_responses/test_responses_tool_use_streaming.yaml b/tests/cassettes/test_openai_responses/test_responses_tool_use_streaming.yaml index ca0561d60..c20fa5ef7 100644 --- a/tests/cassettes/test_openai_responses/test_responses_tool_use_streaming.yaml +++ b/tests/cassettes/test_openai_responses/test_responses_tool_use_streaming.yaml @@ -42,89 +42,89 @@ interactions: body: string: 'event: response.created - data: {"type":"response.created","response":{"id":"resp_044ee23f1de77c370169fa9f50e844819199db824931af4641","object":"response","created_at":1778032464,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Multiply + data: {"type":"response.created","response":{"id":"resp_00d64fa806f333310169fab1be69d081a08f8285661855594c","object":"response","created_at":1778037182,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Multiply two numbers.","name":"multiply","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} event: response.in_progress - data: {"type":"response.in_progress","response":{"id":"resp_044ee23f1de77c370169fa9f50e844819199db824931af4641","object":"response","created_at":1778032464,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Multiply + data: {"type":"response.in_progress","response":{"id":"resp_00d64fa806f333310169fab1be69d081a08f8285661855594c","object":"response","created_at":1778037182,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Multiply two numbers.","name":"multiply","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} event: response.output_item.added - data: {"type":"response.output_item.added","item":{"id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","type":"function_call","status":"in_progress","arguments":"","call_id":"call_LHDmxFdi7rwbMbWoZc2CdkHs","name":"multiply"},"output_index":0,"sequence_number":2} + data: {"type":"response.output_item.added","item":{"id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","type":"function_call","status":"in_progress","arguments":"","call_id":"call_sVidsfFJ6zlzRpelrPkTPlpd","name":"multiply"},"output_index":0,"sequence_number":2} event: response.function_call_arguments.delta - data: {"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"TSJ31mxTQGPOro","output_index":0,"sequence_number":3} + data: {"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"ReLAoBL6oNRsXA","output_index":0,"sequence_number":3} event: response.function_call_arguments.delta - data: {"type":"response.function_call_arguments.delta","delta":"a","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"YVpNcmj2Vt5N34c","output_index":0,"sequence_number":4} + data: {"type":"response.function_call_arguments.delta","delta":"a","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"oZA1PK3x65X0w51","output_index":0,"sequence_number":4} event: response.function_call_arguments.delta - data: {"type":"response.function_call_arguments.delta","delta":"\":","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"rK5gXnCa0e2INo","output_index":0,"sequence_number":5} + data: {"type":"response.function_call_arguments.delta","delta":"\":","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"WFzXP9CAvnsSdE","output_index":0,"sequence_number":5} event: response.function_call_arguments.delta - data: {"type":"response.function_call_arguments.delta","delta":"123","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"v10PcJ1qDR8m7","output_index":0,"sequence_number":6} + data: {"type":"response.function_call_arguments.delta","delta":"123","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"3bk9yszS1TEHx","output_index":0,"sequence_number":6} event: response.function_call_arguments.delta - data: {"type":"response.function_call_arguments.delta","delta":"1","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"iGpOyjZcI5jzNIp","output_index":0,"sequence_number":7} + data: {"type":"response.function_call_arguments.delta","delta":"1","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"ckqcfb2NtXm34XT","output_index":0,"sequence_number":7} event: response.function_call_arguments.delta - data: {"type":"response.function_call_arguments.delta","delta":",\"","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"sCuCqOYfD3Hnyi","output_index":0,"sequence_number":8} + data: {"type":"response.function_call_arguments.delta","delta":",\"","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"cspustnrzHoiSV","output_index":0,"sequence_number":8} event: response.function_call_arguments.delta - data: {"type":"response.function_call_arguments.delta","delta":"b","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"7tizScBl6S0jyhq","output_index":0,"sequence_number":9} + data: {"type":"response.function_call_arguments.delta","delta":"b","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"HhO4w36skjtIxaU","output_index":0,"sequence_number":9} event: response.function_call_arguments.delta - data: {"type":"response.function_call_arguments.delta","delta":"\":","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"aTDdMGY2Tk7qnV","output_index":0,"sequence_number":10} + data: {"type":"response.function_call_arguments.delta","delta":"\":","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"SnDoVlFiXfYxtl","output_index":0,"sequence_number":10} event: response.function_call_arguments.delta - data: {"type":"response.function_call_arguments.delta","delta":"233","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"kg7VGPdV93SyR","output_index":0,"sequence_number":11} + data: {"type":"response.function_call_arguments.delta","delta":"233","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"arziGMMxMM5qo","output_index":0,"sequence_number":11} event: response.function_call_arguments.delta - data: {"type":"response.function_call_arguments.delta","delta":"1","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"FD8tdtuvZ0Q9NLW","output_index":0,"sequence_number":12} + data: {"type":"response.function_call_arguments.delta","delta":"1","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"obhmaTA5ZilaAQc","output_index":0,"sequence_number":12} event: response.function_call_arguments.delta - data: {"type":"response.function_call_arguments.delta","delta":"}","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","obfuscation":"xuOXpmYM4HPWedI","output_index":0,"sequence_number":13} + data: {"type":"response.function_call_arguments.delta","delta":"}","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"Mezf0KdDroRLDUI","output_index":0,"sequence_number":13} event: response.function_call_arguments.done - data: {"type":"response.function_call_arguments.done","arguments":"{\"a\":1231,\"b\":2331}","item_id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","output_index":0,"sequence_number":14} + data: {"type":"response.function_call_arguments.done","arguments":"{\"a\":1231,\"b\":2331}","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","output_index":0,"sequence_number":14} event: response.output_item.done - data: {"type":"response.output_item.done","item":{"id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","type":"function_call","status":"completed","arguments":"{\"a\":1231,\"b\":2331}","call_id":"call_LHDmxFdi7rwbMbWoZc2CdkHs","name":"multiply"},"output_index":0,"sequence_number":15} + data: {"type":"response.output_item.done","item":{"id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","type":"function_call","status":"completed","arguments":"{\"a\":1231,\"b\":2331}","call_id":"call_sVidsfFJ6zlzRpelrPkTPlpd","name":"multiply"},"output_index":0,"sequence_number":15} event: response.completed - data: {"type":"response.completed","response":{"id":"resp_044ee23f1de77c370169fa9f50e844819199db824931af4641","object":"response","created_at":1778032464,"status":"completed","background":false,"completed_at":1778032465,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[{"id":"fc_044ee23f1de77c370169fa9f51aeac819184801e96c85b1397","type":"function_call","status":"completed","arguments":"{\"a\":1231,\"b\":2331}","call_id":"call_LHDmxFdi7rwbMbWoZc2CdkHs","name":"multiply"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Multiply + data: {"type":"response.completed","response":{"id":"resp_00d64fa806f333310169fab1be69d081a08f8285661855594c","object":"response","created_at":1778037182,"status":"completed","background":false,"completed_at":1778037183,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[{"id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","type":"function_call","status":"completed","arguments":"{\"a\":1231,\"b\":2331}","call_id":"call_sVidsfFJ6zlzRpelrPkTPlpd","name":"multiply"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Multiply two numbers.","name":"multiply","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":58,"input_tokens_details":{"cached_tokens":0},"output_tokens":23,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":81},"user":null,"metadata":{}},"sequence_number":16} @@ -135,15 +135,15 @@ interactions: cf-cache-status: - DYNAMIC cf-ray: - - 9f745b59588a0d35-ORD + - 9f74ce85bdf0cb75-DFW content-type: - text/event-stream; charset=utf-8 date: - - Wed, 06 May 2026 01:54:25 GMT + - Wed, 06 May 2026 03:13:02 GMT openai-organization: - user-r3e61fpak04cbaokp5buoae4 openai-processing-ms: - - '262' + - '266' openai-project: - proj_2Y4H7T2XPShczTQx5fqMXdNp openai-version: @@ -151,9 +151,9 @@ interactions: server: - cloudflare set-cookie: - - __cf_bm=6VXg_BM_1a0kiY0dxVuH8m4VdhuXjNNXVwe8H1X6TnI-1778032464.8521352-1.0.1.1-W9VN6MIUcIqwnCiXpBplvoBRsAGZFB7KzfpnSev8YLeCWnfvp6yEJM5D.ekb1ubddrlBac.xiewMcEfuKRCpRGPnI.AIr3QCYjyGncQtKLAtCRIew34WL.a3qZBuwD0b; + - __cf_bm=dOVwLeU7r7aShRQaxTKiYcu2Y1q8ZJiD8OFt516IQkk-1778037182.353376-1.0.1.1-kDgOFNW1kiTQunazntKDBwpmZh4ai7MGVZI2ZIDLzRXP4MRBfrPevoYSr41Ul_ozGR_VudKgENypl8pRCVaMoqfP66IEVu.Oz7Jz1Le7WzsMCL78NWXm.AbJCZy_KKVD; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 - 02:24:25 GMT + 03:43:02 GMT strict-transport-security: - max-age=31536000; includeSubDomains; preload transfer-encoding: @@ -173,14 +173,14 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_465f47d6541b4fe1963395faa1bf2739 + - req_996657ef97514aeb93f449d57303514d status: code: 200 message: OK - request: body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"What - is 1231 * 2331? Use the multiply tool."},{"type":"function_call","call_id":"call_LHDmxFdi7rwbMbWoZc2CdkHs","name":"multiply","arguments":"{\"a\": - 1231, \"b\": 2331}"},{"type":"function_call_output","call_id":"call_LHDmxFdi7rwbMbWoZc2CdkHs","output":"2869461"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":true,"tools":[{"type":"function","name":"multiply","description":"Multiply + is 1231 * 2331? Use the multiply tool."},{"type":"function_call","call_id":"call_sVidsfFJ6zlzRpelrPkTPlpd","name":"multiply","arguments":"{\"a\": + 1231, \"b\": 2331}"},{"type":"function_call_output","call_id":"call_sVidsfFJ6zlzRpelrPkTPlpd","output":"2869461"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":true,"tools":[{"type":"function","name":"multiply","description":"Multiply two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}]}' headers: Accept: @@ -220,8 +220,8 @@ interactions: response: body: string: "event: response.created\ndata: {\"type\":\"response.created\",\"response\"\ - :{\"id\":\"resp_06c574f2869625a90169fa9f523b2c81a1a08f987e0980156e\",\"object\"\ - :\"response\",\"created_at\":1778032466,\"status\":\"in_progress\",\"background\"\ + :{\"id\":\"resp_0dacb603de1c9e6b0169fab1c2314081a3b1df3cc5c09e0c60\",\"object\"\ + :\"response\",\"created_at\":1778037186,\"status\":\"in_progress\",\"background\"\ :false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\"\ :null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\"\ :null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[],\"\ @@ -236,8 +236,8 @@ interactions: ,\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\"\ :0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\"\ :{}},\"sequence_number\":0}\n\nevent: response.in_progress\ndata: {\"type\"\ - :\"response.in_progress\",\"response\":{\"id\":\"resp_06c574f2869625a90169fa9f523b2c81a1a08f987e0980156e\"\ - ,\"object\":\"response\",\"created_at\":1778032466,\"status\":\"in_progress\"\ + :\"response.in_progress\",\"response\":{\"id\":\"resp_0dacb603de1c9e6b0169fab1c2314081a3b1df3cc5c09e0c60\"\ + ,\"object\":\"response\",\"created_at\":1778037186,\"status\":\"in_progress\"\ ,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\"\ :0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\"\ :null,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\"\ @@ -252,80 +252,80 @@ interactions: ],\"type\":\"object\",\"additionalProperties\":false},\"strict\":true}],\"\ top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"\ user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: response.output_item.added\n\ - data: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + data: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ ,\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"phase\"\ :\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\"\ :2}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\"\ - ,\"content_index\":0,\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"content_index\":0,\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ ,\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"\ logprobs\":[],\"text\":\"\"},\"sequence_number\":3}\n\nevent: response.output_text.delta\n\ data: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\"\ - :\"123\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ - ,\"logprobs\":[],\"obfuscation\":\"PsjcK87GfnMGs\",\"output_index\":0,\"sequence_number\"\ + :\"123\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"yJXGKTBmb2UOz\",\"output_index\":0,\"sequence_number\"\ :4}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ - ,\"content_index\":0,\"delta\":\"1\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ - ,\"logprobs\":[],\"obfuscation\":\"wcAoCtBmb6T7zWp\",\"output_index\":0,\"\ + ,\"content_index\":0,\"delta\":\"1\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"0bPnrPoF6d7HddV\",\"output_index\":0,\"\ sequence_number\":5}\n\nevent: response.output_text.delta\ndata: {\"type\"\ :\"response.output_text.delta\",\"content_index\":0,\"delta\":\" \xD7\",\"\ - item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\",\"logprobs\"\ - :[],\"obfuscation\":\"5cOLfvSirBOxck\",\"output_index\":0,\"sequence_number\"\ + item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"logprobs\"\ + :[],\"obfuscation\":\"ft7PFRXyUN31Dr\",\"output_index\":0,\"sequence_number\"\ :6}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ - ,\"content_index\":0,\"delta\":\" \",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ - ,\"logprobs\":[],\"obfuscation\":\"yDlX8TgyBiqvfBc\",\"output_index\":0,\"\ + ,\"content_index\":0,\"delta\":\" \",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"HWsKMbb1JWpQBGo\",\"output_index\":0,\"\ sequence_number\":7}\n\nevent: response.output_text.delta\ndata: {\"type\"\ :\"response.output_text.delta\",\"content_index\":0,\"delta\":\"233\",\"item_id\"\ - :\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\",\"logprobs\":[],\"\ - obfuscation\":\"lwWcyMe8rNUNP\",\"output_index\":0,\"sequence_number\":8}\n\ + :\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"logprobs\":[],\"\ + obfuscation\":\"OyardqgvtiISP\",\"output_index\":0,\"sequence_number\":8}\n\ \nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ - ,\"content_index\":0,\"delta\":\"1\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ - ,\"logprobs\":[],\"obfuscation\":\"N6xkEhKWhnK9zYZ\",\"output_index\":0,\"\ + ,\"content_index\":0,\"delta\":\"1\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"M5u2aOAAbIF9HaL\",\"output_index\":0,\"\ sequence_number\":9}\n\nevent: response.output_text.delta\ndata: {\"type\"\ :\"response.output_text.delta\",\"content_index\":0,\"delta\":\" =\",\"item_id\"\ - :\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\",\"logprobs\":[],\"\ - obfuscation\":\"fw7O0adbH3ppH9\",\"output_index\":0,\"sequence_number\":10}\n\ + :\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"logprobs\":[],\"\ + obfuscation\":\"DblLLi7wbRu7Yj\",\"output_index\":0,\"sequence_number\":10}\n\ \nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ - ,\"content_index\":0,\"delta\":\" **\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ - ,\"logprobs\":[],\"obfuscation\":\"MYjlLyYLXMlda\",\"output_index\":0,\"sequence_number\"\ + ,\"content_index\":0,\"delta\":\" **\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"iiojCebBdYGkO\",\"output_index\":0,\"sequence_number\"\ :11}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ - ,\"content_index\":0,\"delta\":\"2\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ - ,\"logprobs\":[],\"obfuscation\":\"lR21X59BeOsCbaf\",\"output_index\":0,\"\ + ,\"content_index\":0,\"delta\":\"2\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"eJPGA9ScIqJTAvj\",\"output_index\":0,\"\ sequence_number\":12}\n\nevent: response.output_text.delta\ndata: {\"type\"\ :\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\"\ - :\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\",\"logprobs\":[],\"\ - obfuscation\":\"MFmjQhCxnhZoDZJ\",\"output_index\":0,\"sequence_number\":13}\n\ + :\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"logprobs\":[],\"\ + obfuscation\":\"pkBo5DcWhNmQVlb\",\"output_index\":0,\"sequence_number\":13}\n\ \nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ - ,\"content_index\":0,\"delta\":\"869\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ - ,\"logprobs\":[],\"obfuscation\":\"wLjwhrxjSc1uy\",\"output_index\":0,\"sequence_number\"\ + ,\"content_index\":0,\"delta\":\"869\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"6Z8oUB1RMn3DD\",\"output_index\":0,\"sequence_number\"\ :14}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ - ,\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ - ,\"logprobs\":[],\"obfuscation\":\"LfxW516uVZ5YUor\",\"output_index\":0,\"\ + ,\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"Bf7uSiaON5ThxbH\",\"output_index\":0,\"\ sequence_number\":15}\n\nevent: response.output_text.delta\ndata: {\"type\"\ :\"response.output_text.delta\",\"content_index\":0,\"delta\":\"461\",\"item_id\"\ - :\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\",\"logprobs\":[],\"\ - obfuscation\":\"oFk25jkAOAbD6\",\"output_index\":0,\"sequence_number\":16}\n\ + :\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"logprobs\":[],\"\ + obfuscation\":\"qqwxt4JuKvQU6\",\"output_index\":0,\"sequence_number\":16}\n\ \nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ - ,\"content_index\":0,\"delta\":\"**\",\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ - ,\"logprobs\":[],\"obfuscation\":\"F0z6oUjKY4ZGlI\",\"output_index\":0,\"\ + ,\"content_index\":0,\"delta\":\"**\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"XM0rK6J7j52uLE\",\"output_index\":0,\"\ sequence_number\":17}\n\nevent: response.output_text.done\ndata: {\"type\"\ - :\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + :\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ ,\"logprobs\":[],\"output_index\":0,\"sequence_number\":18,\"text\":\"1231\ \ \xD7 2331 = **2,869,461**\"}\n\nevent: response.content_part.done\ndata:\ \ {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\"\ - :\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\",\"output_index\"\ + :\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"output_index\"\ :0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"\ text\":\"1231 \xD7 2331 = **2,869,461**\"},\"sequence_number\":19}\n\nevent:\ \ response.output_item.done\ndata: {\"type\":\"response.output_item.done\"\ - ,\"item\":{\"id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + ,\"item\":{\"id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ ,\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\"\ ,\"annotations\":[],\"logprobs\":[],\"text\":\"1231 \xD7 2331 = **2,869,461**\"\ }],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"\ sequence_number\":20}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\"\ - ,\"response\":{\"id\":\"resp_06c574f2869625a90169fa9f523b2c81a1a08f987e0980156e\"\ - ,\"object\":\"response\",\"created_at\":1778032466,\"status\":\"completed\"\ - ,\"background\":false,\"completed_at\":1778032467,\"error\":null,\"frequency_penalty\"\ + ,\"response\":{\"id\":\"resp_0dacb603de1c9e6b0169fab1c2314081a3b1df3cc5c09e0c60\"\ + ,\"object\":\"response\",\"created_at\":1778037186,\"status\":\"completed\"\ + ,\"background\":false,\"completed_at\":1778037187,\"error\":null,\"frequency_penalty\"\ :0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\"\ :null,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\"\ - :null,\"output\":[{\"id\":\"msg_06c574f2869625a90169fa9f52f7f481a1841d5b486171d57d\"\ + :null,\"output\":[{\"id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ ,\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\"\ ,\"annotations\":[],\"logprobs\":[],\"text\":\"1231 \xD7 2331 = **2,869,461**\"\ }],\"phase\":\"final_answer\",\"role\":\"assistant\"}],\"parallel_tool_calls\"\ @@ -348,15 +348,15 @@ interactions: cf-cache-status: - DYNAMIC cf-ray: - - 9f745b61ac4a0044-ORD + - 9f74ce9d5cec11fd-ORD content-type: - text/event-stream; charset=utf-8 date: - - Wed, 06 May 2026 01:54:26 GMT + - Wed, 06 May 2026 03:13:06 GMT openai-organization: - user-r3e61fpak04cbaokp5buoae4 openai-processing-ms: - - '310' + - '353' openai-project: - proj_2Y4H7T2XPShczTQx5fqMXdNp openai-version: @@ -364,9 +364,9 @@ interactions: server: - cloudflare set-cookie: - - __cf_bm=FINbeYv4P8EmjyVhAWYLQKgBylkGZZfMLrWBueXEPEU-1778032466.1842103-1.0.1.1-Z5aNN38FYUcK1gh7SXuH81x1g73cX2Etv1nZkiRDxuaShelUUtL7xi_DUs3aRin22APa4.qbzj3.HOslue0ntVZ2.Bqyt5gPyMnFk3o9pVo8LnBCJgKKR7BFDhbOtKS2; + - __cf_bm=YuSyQqrRg_ripxVBGX9ZnLTFASaREsd8RjcORfPvRSE-1778037186.1373405-1.0.1.1-OTt6GIIb.vqG7KdooMAgro_13UWD_y3xibuJnSKYjofIRRPgkkgl3ivm9KWDf85p3Bq6X.FzVkbHHpoqVRfiGNNvdSjs_gxjjGTIEXa8..L8k11d2g6DXPHMJLp1bXFs; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 - 02:24:26 GMT + 03:43:06 GMT strict-transport-security: - max-age=31536000; includeSubDomains; preload transfer-encoding: @@ -386,7 +386,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_87194de4a3a744a78ed7904a96cd1e5e + - req_808d2a0863014e248d080b21301d5d58 status: code: 200 message: OK diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index 84edb0f2c..bbaa94c4f 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -217,3 +217,56 @@ def multiply(a: int, b: int) -> int: assert "2869461" in output.replace(",", "") first, second = chain._responses assert first.tool_calls()[0].arguments == {"a": 1231, "b": 2331} + + +@pytest.mark.vcr +def test_responses_round_trips_encrypted_reasoning(vcr): + """Reasoning items returned by the API in the first turn must be + echoed back verbatim on the second turn so the model can pick up + its hidden chain of thought after the tool result arrives.""" + from llm.parts import ReasoningPart + + model = llm.get_model("gpt-5.5") + + def lookup_population(country: str) -> int: + "Returns the current population of the specified fictional country." + return 123124 + + def can_have_dragons(population: int) -> bool: + "Returns True if the specified population can have dragons." + return population > 10000 + + chain = model.chain( + "Pick a clever country name, look up its population, then check " + "whether it can have dragons. Be brief.", + tools=[lookup_population, can_have_dragons], + stream=False, + options={"reasoning_effort": "high"}, + key=API_KEY, + ) + chain.text() # drain the chain + + first = chain._responses[0] + + # The first response must produce at least one ReasoningPart carrying + # the opaque encrypted_content + id. + reasoning_parts = [ + p for m in first.messages() for p in m.parts if isinstance(p, ReasoningPart) + ] + assert reasoning_parts, "first turn should expose at least one ReasoningPart" + pm = reasoning_parts[0].provider_metadata or {} + assert "openai" in pm + assert pm["openai"].get("encrypted_content"), "encrypted_content must be captured" + assert pm["openai"].get("id"), "reasoning id must be captured" + + # The second turn's outgoing input must echo back that reasoning + # item, otherwise the model loses its chain of thought. + second = chain._responses[1] + second_input = (second._prompt_json or {}).get("input") or [] + reasoning_inputs = [it for it in second_input if it.get("type") == "reasoning"] + assert reasoning_inputs, "second turn must echo a reasoning input item" + assert ( + reasoning_inputs[0]["encrypted_content"] + == pm["openai"]["encrypted_content"] + ) + assert reasoning_inputs[0]["id"] == pm["openai"]["id"] From 1a56805ceb59d3e7338cc9dbb780385126154a60 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 04:14:04 +0000 Subject: [PATCH 075/258] Verify tool calls during reasoning work end-to-end The previous commit wired up encrypted_content round-trip but only tested that the data flows through correctly on a single tool round- trip. This adds a multi-turn cassette test that proves the full interleaved-reasoning capability: - Each turn produces fresh reasoning_tokens (not just the first) - Every prior reasoning block is round-tripped on every subsequent turn (the Nth turn echoes >= N-1 reasoning items) - ReasoningParts persisted on the assistant messages carry the same encrypted_content + id that gets sent back on the wire The puzzle is shaped so the model can't parallelize tool calls - each db_lookup result tells it the next key to use, forcing the model to think between calls. The recorded 4-turn chain shows reasoning_tokens of 45/98/196/17 across turns with reasoning items accumulating in every outgoing input. This is the GPT-5-class capability that Chat Completions can't deliver because it discards reasoning between turns. --- ...erleaved_reasoning_between_tool_calls.yaml | 540 ++++++++++++++++++ tests/test_openai_responses.py | 105 ++++ 2 files changed, 645 insertions(+) create mode 100644 tests/cassettes/test_openai_responses/test_responses_interleaved_reasoning_between_tool_calls.yaml diff --git a/tests/cassettes/test_openai_responses/test_responses_interleaved_reasoning_between_tool_calls.yaml b/tests/cassettes/test_openai_responses/test_responses_interleaved_reasoning_between_tool_calls.yaml new file mode 100644 index 000000000..ec692703c --- /dev/null +++ b/tests/cassettes/test_openai_responses/test_responses_interleaved_reasoning_between_tool_calls.yaml @@ -0,0 +1,540 @@ +interactions: +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Solve + this puzzle: call db_lookup(''start''), then follow each instruction step by + step. Each lookup tells you the next key to use. Compute each step in your head. + State only the final integer."}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"db_lookup","description":"Look + up a value by key in the puzzle database.","parameters":{"properties":{"key":{"type":"string"}},"required":["key"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '551' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/31WWbOaShB+z6+wfA4pQNb7prKouCKuNylqgGGR1ZnBLXX++wXMUc9Jcn2woD+6 + p6e7Z77v55dWqx157X9abQRxYdMcK7uM7/q86/igw9OMIPvA8SHf4UVXYmSxehYAgILvcLzvO3L7 + ax0idw7QJe9h8gzDu91FEBDo2aDGGFGUaI6WaL7BMAGkxLWPm6dFAqvv7k4OcOMA5WVW5+WDBMO7 + OUqSKAsq28/qtTIU4ApR7e/BE0zyonqpgLf7wu8hPy0tNihEKK89szJJGoOP4LGEmXu1C5iBhFwr + kP5GN1iUvQezPUhAlOBXzyjDBJUuiapNv9pTcLHzkhQlsUkew99BkueJ7YLkY7g092BS7ykoCMV/ + 4ymWZgWK5ii20358gEC93KvbfaXK8m9TmnuBnq3Ff2+swAK5aazMcIIoQiADlmM5QDfLNUHItYD3 + 1gKcZ3ULHlBVMnQt6jK7eUZg1oxA0K1/vYJCdqaommIXY0+YqLd81telmQHjI6WepFPmzK14GUHO + zE99E2+Zsot2OKWvhnHtdtVdKBHztl8bKx8anjCwdupRR+cVHMpTTzQYrtQQve5LdlfVZlveNtPo + Fo6G2cGfjHv8BbHCMhg6y7i3K9Sxvrvl3IDKku5ofc5sK8giygD8aXoY7XHIUWf+vNouOlNgcAfa + 2Y52ypkYWrLgk/FtDjgjuMWbUSzQg4shceKe369jcYMH8tJQlPlyrGKXpNvLjj+Wo1jerpb6lI9N + 2pobOnaHF2tLbsp84JZeP+oePFM/0YFtDUfJcLUQcl/ZbhaJ5HjFLj8uho567RjhMezE2MiXushk + 9AqvzpTg0UPYmyxSmlu7etRfhcoAGKCjhtol5CfTIC3X6kBZFlzveobrsyHqnJ5FhidNTzEotas5 + W7Cz9JgdbGFigI3rDsFinfK7s7xZ0fZ8ONeUmXHFUHRTzJLkqhfaBSqBFSy7Jr6MwV7dT4YbN9pb + jAJuXXmMkMJ0QF/h4Hyw6Zsjp3BiZSBqk47FWshiBEqn435U2LNtKmv7/WV06gar7cTJ1ipmlmWw + 6fXZzOr7OzQ2z9vD9baPOzyMbydhPLyK+ygIaULZpjXboF48vXQp8XAK+nN7p/GMqzNZzoWWpg7C + 0C3Pu2U6WU4m3djdRLQSUtrWuYqxTA4rfWJYB8yYU3k6lIjLJstz2lW9cO/F9qhQs3XiF4qKD6Zm + DlXB9INQo5ZmxgZad+Wb7g0OjCJJyWZCQ8WIgCFsjYE1XwCC6fTG9BgBJOZopkLNOU/miy4zud5E + QAxH2vNDe4gSvRcyg1x2SrCzTc3vFzpM5+5MhibS8724GJ+Xi4FQnJkJUkq8twvCnYxdv3NUyg5S + 97YwG5y1XWbO1wd7UXIOkX0eIGNuGd2CDVMV74y+cgIb2CvYJFBnjiDgU4c7DoQjpvK8353pIkrJ + YTnur0tJo02GmUaaG51PaZKq/dnZkj0BpoFxyGSLs9l+D6zSXEaeGk39I+Wm4UQXASscdHDDw0Qe + 3qgClRcxXEuSyES7ZKqEmT1BkhDH/TTOjDHfIbpDsfLWNaiDtcqSsY4GaZj1Elro+7RPuaqILUFh + yQG4CgzluRCQkoZZz5875Wgg8nmX5ehBZKzEwOx5VLzLzkJ4CvsTYxUfHElaHXhmps0uPfqkjDON + s4WrhPD5usnnawhLc7IZjEhuQ4TNZVRustv6MOrmnGfIk7HfM8xwSEJho/RKwYRjlhf5hX9b7VTN + 0repPt92OrP1mabSIH1edbhMU4BqYvj3R2N7+/qnu9Z3/37XirTHSPVdK/kCBB4nsjIPKkZlf79r + /TJrOKVhiJck/sKZDQhQUKbVHdzgP7+3Y3j93v7ne+2EyPf22/PLOqh9z7d5HAoinXZvptDd4ZuX + H8FQFo4BHT89MpA2eXmOneR5XBbtew2q/x8NAxUAVZFg8pHXKmq8k3dR6YKKLOAf+LWCTlFeYvtd + Otwze3BbgaqtkiqkG0K72tNfMQRrBrozY5vlwjthPlnrIRyg7+eooaqwumN+bfKlwXX4h5LAwIfk + WuVUx/Yj+EE3YIhOUbUrEr0rER+UCWn/Ejg5gh8kDIFpUbN32diZXwUg8EKeyVWppeD5/jITzXev + o9c+QeTkOGrK2U6hF5XpUwLdGxHmVX41DEqStx8A/l0rfB69Z/M9iF0UFe+lHVcD0CqLFmidQFLC + lnNtVW1pRVmLhLBVlLdbAlseIMABGH77vxl6QPX0pFX7EH7Z+L2/VcGq6n60V8h9EF5NL1uoJFnd + 8Bfw7fH89vUZvlZ9EYLeoxgvwR/vP14c3hf4pXdfEOB5UV0gkMxfc256/+XTyk1+jVyuz8eng0Ty + oipPUO3cqQPQD2PRnBlZuhtQ1aV3Gdj2Igyc5F0/lxgE8DlQUfZBhErS19/tL8r25/OSqA6V93Sk + P4zeZ20r8H8C/hT3cSCfzhz/ITbJCUieKMN3HkNd4o8HsBoaUI9avcDbl7f/ADPaWJDQDAAA + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f7526f32c3c0044-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 04:13:27 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '2091' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=N7TAdwEKwrRihZ4HPzOd8tAdwT5JnZWp1Mj2.UfCC3g-1778040804.348554-1.0.1.1-NFuNpo0J7tlz8j4YDeoThOIY2CDX.4CkvnAcV4AwMaERfiFLSTPjDY8gcSPYiwIXP3TeplBxIS2vJK_VrxoTlp4TAVqwN8bgJ6CMnQFDfKuS7S0zRVqTDUsqYdgW6ccN; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 04:43:27 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_c48ea45fc57a407bb6b100db8c59af68 + status: + code: 200 + message: OK +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Solve + this puzzle: call db_lookup(''start''), then follow each instruction step by + step. Each lookup tells you the next key to use. Compute each step in your head. + State only the final integer."},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfe62a9c8197914677ea9a2424a0","encrypted_content":"gAAAAABp-r_nDEFD_pLd6MEzoOCG8OKekq-Ev8vnbPTkSie4RovCRsX1uArYsm0yKKyAAEYh8tRzZVKUfeKd6HTYEqGrwUeI9Nd7K14uFr0VC8_AEFOX5_RmizhJInjfMLB5xr26SgIbSkBYpELGYzo4H-nlAJVwn_Tgni-Ka5vNjJZsh4-w5wUXQ3NaK4j0bXJYDwtKFlQ5lLzPa4KgzkWJk60HxK847Z5ZVk7WsH9SKDDPSLEsctmXxY5quJk9XUSGN5kR0TPKGscIxTXtzDPHcudCiAjdRGv0g_TIJlIUQ6ofDXWQl8bdpYoqQIbEy3Khqh3ksKoSG71n0UsUw-6d0IeBMQm04VcGiCUhDHaKa3EhFxh5MNgmuVEHDSp4ByweVwK7G4GniKd8NvkauFyROQ2Omqnj_6MKaWccIaQVm5Yw9WU0_PIPFDOKyse7cms2tlyGpFxeDgTgSARsxLaZEZMIWciZT1DazA9LrrD13aCD4ePHWCRJbpbkDH7FM3T2TrT16-G0kCip_OXm9FZZxJvAgUXMbnVEs1SugWBC2nTCfYrLRwXjyzZk35ekzv6LIy7Zigh0t-_RTOWrBkNxA-7jvgCP_YF51cG1no4hTFEHhhcuwYSmMSMMAkcWi0Dh-FXby7k9tjUGMKTjs1RN9NI8tc2lSwmAEdhZdk_JpEnVlfpDEsjRFRIE6RfghF-SRn2gFAUfRczeHKplmtWM0eDKiaK6XKHTPQats0mz1B16alRJOEeFbwMPQA1Myz7atKb8Z5I_IrlGBh1Ho9buaY_RFfCpGemPcO9eRrGoZ7QLwSQH6pw1MrDusZ_pt4vKYC3qDu3rEZ_6OHwFYnRPVj_Qu4bt9f5arKPTKAp2hmEsYKCDvaWeBp2lgEOb66sv34qH6qs-ooCAOG7rmtjSLCVu8F0R11NiFciwvmlmECOwT9d6emgKjn9T4_2CBaUmo9rdEiNfq-cmhMG7a26jGazsIl9Iz-prux7hV8871iYlNDhn_Mr86kkCmknKL53tGb-29XcK-jTUnlLGrHmhnBl06Cf0f-cE7sT6D2tjacDeh9P6gtu0enBfPbuJH75oA240HiKU7gRBd-kYnw6hvhCMKUkjb88Uj51OFOxB0vDLnF4_6y8rswyWoPVeeuRMWHJto_ersRSiuWnzVjJAo4dK9MLfBKRhIth6WDBu6ReL2575QfzUYEFTGXmGPX33OVw0-mgm","summary":[]},{"type":"function_call","call_id":"call_I670mAzR6AYszdoqaI96qg0k","name":"db_lookup","arguments":"{\"key\": + \"start\"}"},{"type":"function_call_output","call_id":"call_I670mAzR6AYszdoqaI96qg0k","output":"Begin + with the value 7."}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"db_lookup","description":"Look + up a value by key in the puzzle database.","parameters":{"properties":{"key":{"type":"string"}},"required":["key"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '2127' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/31XWbOiyBJ+719xwudmApR1IuZBERBRERG36QmigGKTtQABO85/v6B91NPL5cGg + 8qvMyq3Mj+9f3t4GgTP4+22AYJGZODnkbMK1Xcq2XDCicILmXGC5kOFYhmUJjuFI3GHhELcIAto4 + 6Qy+9iZSK4R2+WEmTQp4l9sIghI6JugxgmFYnMRZnLlhRQnKquh17DTOItjtuytZwD57KK2S3i8X + RAW8i4MoChKvk33vlp0gAy1Evb4DLzBKs27RAe/3gz9Mfj6awG8oRCjtNZMqim4CF8G8gondmhlM + QFS2HYj/dd8cJB/GTAeWIIiKV80gKUpU2WXQBf0qj0FjplWZVaVZpmf4K1imaWTaIPpsLk4dGPUx + eVmJUX9R2BAf0hhOYsPR4LEBgf64V7X7SZ3k31tq7gl6lrb4c2HZEcneCsv2DwDAtYbAItx7LW5G + yjaD99KCIk36EjygLmWozfo022lSwuTWAt64fyYZhswcKDNWwsY4yyrr2Bwr5517viLBrLlQCpxl + vvBH7iEFNSdb5VjnprVvmCqf+F4ocuV0pNB0E6Md6e0u0wMVu2FYiOSYRYgrnOO6XNbENrbrZVJO + dsplXpyVMAtPkC4op6iTK3+tjGsl0YaqZJssPWVJzMuwpvQYO6XNEV+H5eEYMubRuZ6vh9hil0Ri + iW0be3KUgilAMDoihhCxui1acyI367ErVpHN8HBrhtKRzrchdgAHIVzBSzY+bxqtGOVTsXCJy3w+ + 15cbo6IuM1HFzrFmXHiCZ5KVflawsdaIpOc3Q9m0pflKn1VFEko5JYMRx8xMnJ+oWrh1iNlcrkbG + XmK9aqmHMy4ewmixFI3MFZqLtSEDBYswWPP2PpNbd984iz2ajuR9HmuU7LCJIM9FhWHjkNzFi3XN + ztX1eYwSW7e9Q75V6KqLWzgMZ2WxEvhG3FA1AXcnnWiDRqBVvVOjrth0GjVNg2/TwBoTcy63C0pT + NhhuLBpzoZhrz+XGUGHN01DMZ6tE9KlE2e2dVj4emm1WoUwkciFRJrzO10BjDldjHs2xhDBE71Ti + xka0Cis/jg6Fz5MVQpe8qNc5sV5Mcl+zT83aKg75UrmKVb2l5zQvREMJGSkfzqzdNJd9IqQjm5pM + jwED1PExYSyh1mSz9N1YswJGQGB3WsaeJSSX47m5XOGaYcc6Tx6DY+3JV6kyNOVa5xtFsZoM0inc + CQwn8vDgC5ayHZ7wsz056UYaN67OB4tyteTxVcDtRddQK46MNuV27axnbLarMG0iwmvASm6jHmar + fFWJ+k42d1p3M+R0TQaAgMfGkod+Vo+j86hiHUPnL/NWdxaLmnPW+5XW2ClFXKcgbrWdAhMrEayQ + POWnjRJhw9JMoISP7QOVRwGfT8dknMcLhmwDhOJjSXNnWdRoXeLXGEuf4xGIvbGcqIU62emagmEz + oF4mhru1MX6/UNVV7oZDcDCCJa9c9OUl5TBarDySL+SDY3tNrmpHybUpNbMhG28NB+lh7FvEPliq + dHTGFzqa5bhsFLLPThPuQFl+V8WEdXiqjpgjYCap23pDYSJtU1K3mQBL3OHEFQ/MOd/GqrGRABmc + IOuvsiptS283jaywjVx81RqYJOBeXVmiF1v0NE8Yw5lDRzADcqrIdlllQZZBO64zT+V2RBJE2+iY + ro4hklAbbjcneiyEbArgZrgZSSRRG9lydqEroz1uZSUioqE9dmhbwj1qzAq8x1maKxv2eGxrq5Hg + uKmcqgdVyPklFvNCGK65WrMb8lquPNxMze2W5vOTt1am1+uepYzOda/cS75B0qS+Foz02MwRn+2D + JtdMdzteOSJZJOuRv5FIpgWVDBPRUUfEtQKuOkd2LkxbW3Gj6ewypo6GGihEoKZThQSW5p8mcB1n + +1YVXTaq3MJYSCks82Ahkbsl5TtBKOdnWd0HTqDok2VzYA8MZY3hlqr0UcLndKLj4WaxVebMYsHl + MxqPoYNxkntdNsohWc4CvIW1t4s11asDRKKCTYdVGRa+hVQ4vZLYCSMzvjjXp9wR8iOu/fPPcwQU + VRwD1A/Mf/+7yd6//m4GufafZxAYWjbZzyBAUCTrOC5OugRLctSvM8itktusvU3OFyf+wCVuIEBe + FXez6YZ//zY4w/bb4O9vA+bb4P25qzdo3n29vRohhdOBoIST0x7N1vmQH0nT0kNPjQTEN58cy4zS + 9Fxlg3v83e9/t6mcAdRZgtHnWd/RhTuhyTqu1A1Q+BvO0UGXIK0K84NO3T17zPsMdWGWnUnbh2YX + zx8xBPupfGcLgyHp30nEc5I/yBR03RTdxrcfeP6PIF+K25t/sKsCuLBsO596224AP3GpAqJL0EVV + Bh/szAVVVA5+kL4UwU+0roRx1jOa6iYnfiSghE35dK5zLQbP9Us/3Pa9tt3gApGVFsEtnYOu0YMq + ftLCeyH8tPOvh0FVpoMHUPzKn35uu2fxHVjYKMg+UrvoGuCtyt7A2wVEFXyz2reuLG9B8lb68C2r + rtcIvjmgBBYo4F//r4ceUN89cVc+VLwEfq9vl7Auu5/lHXJvhFfRSwgdTe0L/gK+P97fvz7N90w4 + QNB5JOPF+GP934vCxwE/vgFeEOA4QZ8gEK1ffb7V/stPJ9/8u31C9Pfjp4tUplmXHq+L3OoN4A9h + drszHHsXoK5KH9R44AQFsKKPb4qqAB58NlSQfCLmBEN8/RV4ofvfn/8S3a1ynpr4p977mfATBPs7 + 5HeGH1fyqc2xn4yXaQmiJzpkuUdbV8XnK9i1DeibrT/g/cv7/wBdVsXR5g0AAA== + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f7527072b8a0044-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 04:13:30 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '2766' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=DRZquxDLKhKw.kobZpdPpYdyAORTxb8d3bk6OV6L_3g-1778040807.5506942-1.0.1.1-C63sdNCp_e3AcsWa9mPoKJLTMlDCSogmc69aY3uzn3tTS0hpqMcaswl4Vlx9jLN79I4aGoL9PKji0tX10HxiuZxf4oEWmMjN_QopXLRhPxlCyZuOq26Bg0WQvU0arTlt; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 04:43:30 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_af2b96dc465344fbaecb33d287edd1c1 + status: + code: 200 + message: OK +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Solve + this puzzle: call db_lookup(''start''), then follow each instruction step by + step. Each lookup tells you the next key to use. Compute each step in your head. + State only the final integer."},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfe62a9c8197914677ea9a2424a0","encrypted_content":"gAAAAABp-r_nDEFD_pLd6MEzoOCG8OKekq-Ev8vnbPTkSie4RovCRsX1uArYsm0yKKyAAEYh8tRzZVKUfeKd6HTYEqGrwUeI9Nd7K14uFr0VC8_AEFOX5_RmizhJInjfMLB5xr26SgIbSkBYpELGYzo4H-nlAJVwn_Tgni-Ka5vNjJZsh4-w5wUXQ3NaK4j0bXJYDwtKFlQ5lLzPa4KgzkWJk60HxK847Z5ZVk7WsH9SKDDPSLEsctmXxY5quJk9XUSGN5kR0TPKGscIxTXtzDPHcudCiAjdRGv0g_TIJlIUQ6ofDXWQl8bdpYoqQIbEy3Khqh3ksKoSG71n0UsUw-6d0IeBMQm04VcGiCUhDHaKa3EhFxh5MNgmuVEHDSp4ByweVwK7G4GniKd8NvkauFyROQ2Omqnj_6MKaWccIaQVm5Yw9WU0_PIPFDOKyse7cms2tlyGpFxeDgTgSARsxLaZEZMIWciZT1DazA9LrrD13aCD4ePHWCRJbpbkDH7FM3T2TrT16-G0kCip_OXm9FZZxJvAgUXMbnVEs1SugWBC2nTCfYrLRwXjyzZk35ekzv6LIy7Zigh0t-_RTOWrBkNxA-7jvgCP_YF51cG1no4hTFEHhhcuwYSmMSMMAkcWi0Dh-FXby7k9tjUGMKTjs1RN9NI8tc2lSwmAEdhZdk_JpEnVlfpDEsjRFRIE6RfghF-SRn2gFAUfRczeHKplmtWM0eDKiaK6XKHTPQats0mz1B16alRJOEeFbwMPQA1Myz7atKb8Z5I_IrlGBh1Ho9buaY_RFfCpGemPcO9eRrGoZ7QLwSQH6pw1MrDusZ_pt4vKYC3qDu3rEZ_6OHwFYnRPVj_Qu4bt9f5arKPTKAp2hmEsYKCDvaWeBp2lgEOb66sv34qH6qs-ooCAOG7rmtjSLCVu8F0R11NiFciwvmlmECOwT9d6emgKjn9T4_2CBaUmo9rdEiNfq-cmhMG7a26jGazsIl9Iz-prux7hV8871iYlNDhn_Mr86kkCmknKL53tGb-29XcK-jTUnlLGrHmhnBl06Cf0f-cE7sT6D2tjacDeh9P6gtu0enBfPbuJH75oA240HiKU7gRBd-kYnw6hvhCMKUkjb88Uj51OFOxB0vDLnF4_6y8rswyWoPVeeuRMWHJto_ersRSiuWnzVjJAo4dK9MLfBKRhIth6WDBu6ReL2575QfzUYEFTGXmGPX33OVw0-mgm","summary":[]},{"type":"function_call","call_id":"call_I670mAzR6AYszdoqaI96qg0k","name":"db_lookup","arguments":"{\"key\": + \"start\"}"},{"type":"function_call_output","call_id":"call_I670mAzR6AYszdoqaI96qg0k","output":"Begin + with the value 7."},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfe83488819788888aaafb2ab1fd","encrypted_content":"gAAAAABp-r_qaKH8G-A088KPm_AKkVfkzrE_w9jGidMqLh3fXoaw9IbtAS9DwhU_OCnhgjF9tD3K66xmrV4gVvDX5mfjjsF4A8rr9sdYPtMw1TmcwMntBVKvJskKjpjZe6s5dswnzCzuUzuG6UOKpRpoZpnmCIew5Sm-ZoxY0PjtXYj7_YdzkzXmb8M1nbFyymgIloaDarelYr71F-wysy_BIxPAfFulc7CeT_jGY6qTj-XaXEjNevpAkRxQs3qDFsf1vJJJSMRUu5vHFO-kmQUvC1C7nNSkK-AQxF4ghx2I_cGJNSHusnjGq5Ia397H_0CBOQjTd1HJIu3UWG8guMSjH9m2elLMFUpfExvbR4iK-l-ewCcWpIyfWxdLWrD3IWqmQ5Id8nEIJFK78mj4VmLPw8JOPkArncScgXqTK6uzkzEX2HtsNECxFR5w1eVZS1yixE6OSj4V5z-DDlxxx0ToibA1J9qcs5QKR-0ULx_LK_Pgf9AeK8_Z2FqHNnFh5nKVWdyIYXxTpurpF1qEnKBCSCwaQ7XzUJlJ-n1UFgZt0URFbsbqY3XshC4urrvqswPq1PLBqhQcZxPbsXqMKzFuwT6J6CEl2GrUoCjHbVDqIh1j6lc5BDYi7aOAYn7bEwQI_thfmQbi7EraVZMmgbEnvYkxvzeP78ASC4YiYwgIzGuUQKzwqRKKbxpe6oeVE79FCeXhEbKT2Z0kcBZSUomxfSCiLtNMC0Ni9WFfUOu94lRtTPdPH8pVu-QBFezi8GfxOXHNqNuFSVI_VQtenIoP4ia1eYxbI2hpwAlk3u8dUSCvJySdLLw9dPWNQxco51zDamyQVKenbnEbj4ZqZRKl-2t_neG0AcX5qliCqDA4mqmL74yirrmYt69kIFQ6SGCP-86km3amgAInOsOBVSQK--HaOvBUfTc-CWLOONqfj2aXUiMCKvSMvo9-6Fug4CsIXdcgxqOQYGfc5Opce8mTUdrSjmhb1WiMO6lk0LSrHq0IUsIh8Dn9X5bhXshn8dC5wl7Ya7Bofyg2EBGTo4Sc7i-nf2BfFX7kqTmOURGa4iZe8hNpuoytgVDlbjylf0NyU-GE0gwubFgmb6Dqn7UdJedE_i4DKIctupippecmwpgO9V1nilTlYoNYjrGryjTRZ6AEj8oaeR2R3G41wUpMHv6uUyYTIKl1l2cAd6cG0g5A8ECg9bQfIUcAAcQN3EdfoIoOXOEqCM-mCEjjP9wQcx4ztNg0_o_TT6CqZgPKDzzW85UgVDgtWGhU464SPEUoYxJrCpWixqQ_fTANdF4snP3hRG47yauIenFdO31zuafOJrcqEDycKflDHvA5YUOiK1iOoDK4abQhZBePmpWyOFf8lufsULGoetqiLG4VM5hdijIqkIOWidiKSBMxX8X75bAeT5uS3nCq6nS0jRLTKJ7LL9qH60med-9GfzMxKXnMHi0yewgVmQOgwir4rs8o2utjshbrOeDz4-Z-4pCskwZqdEqY0Q==","summary":[]},{"type":"function_call","call_id":"call_Uj506iEKjBZWrHPq2C3GDtgr","name":"db_lookup","arguments":"{\"key\": + \"7\"}"},{"type":"function_call_output","call_id":"call_Uj506iEKjBZWrHPq2C3GDtgr","output":"unknown + key"}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"db_lookup","description":"Look + up a value by key in the puzzle database.","parameters":{"properties":{"key":{"type":"string"}},"required":["key"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '3967' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/31XWbOiyBJ+n19x4jwPE4CsE3EfVAQERRQQ8M4EwVIg+1Yg0NH//aLO2bp7rg8G + lV9lVm4FX3777eXlNQ5e/3x5bUBbOSiBsz4W+iHpe6G7IFGMYkPXC4HLhB7BYCztoQuMoFHCoxkG + xQPq9fe7idJLgA/fzJRFC55yvwEuBIHj3jGMphmUQBkMfWAtdGHX3nX8Mq8yMO97Knmun0ZN2RV3 + v0I3a8FTHGdZXESz7Nu8nAWVO4Lmrh+AHmRlNS9m4Pvz4DeTPxxNPlDQNOVds+iy7CEIG1B3oPBH + pwKFm8FxBtE/nn7GxZsxJwDQjbP2s2ZctLDpfBjPQX+W5+7glB2sOujAMgU/g7AsM8d3s6/m8jIA + 2T2mqIII+QeJ4ChOISiB4IvX9w2Nez/us9rzpFny30dqngn6KG3774X1SIp9FJah0YUbMiS5cFF6 + 4bGP4x5G4FiBZ2ndtizuJXiH5pQ1Y3VPs18WEBSPFoiW99+qQhqn5wVfM48cyiwmUmpril7nMss6 + l7oubmumoBIiPYfmVRFJia2kGwno7owduyYIEtHGbvQq8M6E2IR6OhGaIcYl5fJA3BuNdzx4XHy6 + qUwqWrik2Yt8cbNca7kqSKeWYobcJfSaoyx6kUx1cKw2QDgi5LRbSO1wQFPS6C4eikFy002dMN0c + dHfC9CxGDelGLBo5Wm9qtxH1rl1k1CLAmMrqZPK0X52QxcUZtuy8B9yGJtVdEeNW0QCzAXLkhQiR + AEVOXihghNbXyFFb3NowIq2mYEhZEm8hM0okuadJfDmslk3vxHV7xuxOvixvIz/oqBjvGu5w2zqC + cxztkENQAxHbswA22lpDIiK5cSpntRB3JVU/jFeodyjnVdZoOEbqNTC7tcThcOmaDDHpXd9bmSkp + AmqaCZVXa19szP3UeUJ61ESG9hHB2mhiuQqG4QQsocy5lojqwlvRgV1FyknpgT3tCul0DDg7DWxR + l7Z0tNqcFX51I282Olmnqw3j/uoIhNLtcN72dbVhJr4d6q1hjTtTwhGWVNCk1xy3Ik6o6GSkbTPw + Jvc60UJ7uWB1XJUhOndHxqLU2SpdKJmJyY7hLimwcAvwfGE2xNm6pqaR0z29JAfySBsHL77CZYGQ + yrU02JWM8f5+Yzuoket+Ziz3iMafm92CgyErwNIudyKjGh6hBdWVHVGO8tyzphSragzOGJOxFoya + ThPJCoWsD4bZsWnSl/neY7uEn5eeU5sXZ8VrfCGVNJ3Z6zKWTaCAzT5b2KVJQKvOyzao6kzjK1Fu + 3AGxBbDnsMKNlPrIbwttByJ/o9VbJkOEsZ5LZS3OwmA6iSpIUXJ1bpeledTlfSz70F8iS5WUDpU0 + uOp0ia6nvVAFW3pRHA762UmO7OGcyrIJEx7su6Djr+YOxrF/bU1TqxnrJnGGrJ9G29nZq2FSeX7Y + Xoe0PuK05xSBpO76kdPZpd4rDEnlgybZIJPRTkqGOsEWB/JibtcVWgWGEG92EYoeMdLYruos4449 + jTk7WKROqW2qxjplXhzy7Y1ndOXMe714JS+84ybAUjwt2oetU2uQFbsLh8PRa7XWFus8hpXboF3g + R4rcj7Eqq5brLdJbrNrbxKGL3s/8en0xjCkI9iMZLDEmaaeNb5MKQQHj7CjwIhzo6xG7kFfNbHJV + A7oh8nLE4McwNxkHJB46snWolsPR10PGms4BHbkKlyZauV8jJN97k+NEgKvSk6fUmtqygN+QJ5nW + 69Rf0aqC14qC5HRSDJHu7kNy6vwlaiW4JiWl7nNrRcPxwKnjdCqRZDp0scTR2mUKNZ3oIikLSW9v + CNv4sIUcAZzZ/3i4XkRzwwiiGCR8UWVu221x3bglhib0kdlQ1GZn5tXt6HKjS3kGDp1rIx7Naxq7 + 0JLsKZ7G7SIzhClEKAVzYtAVm03d7Zrq7AbEjtsVtrAlS48/GpLrNRwCjltECuWTLGjZaQUja23B + WJcimiBSqSfPUhv58pgSQ1KCcXkzlzssv6HKfCWtA5B2gSlss5J2zOZc8QrUisgu0iVHj9ZotqXq + +dWqSp1kgds66V4oRQejTiaiY3F+h1Q1lfJsuw1U6jSitSag1mac5tdGlRveDu3L2zQqplrKVtCa + 9nlnLX1iyyMbZ9k4+dTupXTlLI5u5uGtjge5SRupax2cTYKr1ZJ1FROp/TNRky1ksNo7FqYjib0b + o+Ykb/g4rzYhjZYuw1zo5QkMauzH01LoEGwQ9PVlzwh1AG9ds4uFQ8GYBVsmS62gJ2PDO4N+Yw0/ + WbFEcrz5jiqLNrVSVxmQZdXWL+awlG7TDrILWrqByZ8EUQ43XLB3uP2Z2zq9RIlILpFFx1yXkkwZ + IOcQYUOcNaCpc+sKF8uKC9AHOZ4yokIjskgh03X+oo376UYoq/kLuQ6GvhtSPbjCCqHYzGYYH3Fy + ahj7tKm41OBge+RS6pCs7BpQHktAuThe2S3Ll+eDiiEVcVMkhRtkfJms9XZz2U4cEmG+r+3wCwcV + dp231jIEfkac5CkYpagXLS/kJ2x14t0uaLKUZtdkeq17FzF3DR8OaUaNrLFBMHzNq5Cm2arLqhJo + lneiVBO1nfxiVXwPVZFSCK0Z40Vsz99qOExRcA2otRI0VtkfW2gau6lbaTS1sUdePf7ng1a0XZ67 + zZ2E/ffvh+z777/iNaH/77wG+PjCv/MalgnDmc9QIYr7LoszP/OasCse/O3Bxj458S/89AG6TdTl + M9954N/+ek3B+Nfrn3/NSqDC/nr9/rHzbtR5+vt4PMcpm6dKFjNyW2ieeNCubhXkwodG4eYPvwLP + ycoy7arXZw7m/78fbG9+a86WQPaVQ8409EmUq5mDz8QM/ILLzlAfl13rvNH0p2fvPLJq5lDhbNK/ + AmeO6V+xBtzZ3pOFvuLE9UlOPxjiO0kHYVg2D1p4jaPrP0F+KvDd/Dtrb90QwHH26W47jMEXjt6C + po/nqGD8xvpDt8vg6z/DRNmAL+MCBHl1Z8rdQ479kwAIBvjh3Oxa7n6sP/XEY9/n1nvtQeOVbfxI + 52sOgrjLP8aNZyGu5ezfHXY7WL6+A+3PvPzH1vsofgBav4mrt9Tu5gZ46aoX96V3sw68eOPLXJaX + uHiBV/BSddOUgZfAha7ntuCP/9dD79C9e/K5fE37KfBnfeeEzdn9Kp+RZyN8Fn0KYR5/7gX/BH5/ + f/7++4f5+4QVNyB4T8Yn4+/rvz8pvB3wz2z5CXGDIL4nyM3Uzz4/av/bDyc//HuMpvf78cNFgmU1 + pyeaI/fuBtB3YfW4MyzzFDRzld5Grtcgbl0ve5tVu9aNwEdDxcWXgW+B4r//DHwaI799vCXmWxV8 + aKJfeu/HQRLH6F8hvzL8fiU/tDGW+mIdltDNPmASY9/7umu/3sG5b9x7t91P+P7b9/8Bf9t5rz8Q + AAA= + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f752719988393b1-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 04:13:35 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '4711' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=ADJyoHaKUvEeC3nK2_sSJ7dr9CEJ2NNMqc9USLXclT4-1778040810.495933-1.0.1.1-uG8A.s4fFg.igFwTcj1WtTLUqjC4PfZvnuYaIcoE7CBZKb_G.0rMV93DAq7s2AeJruAKwYTEjGzqEdIZMlC3NnwS6wHlmWG9sXbbAroBBCjQg2JoSGoYK1dMCFoUIdsl; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 04:43:35 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_c6ffb3ca01ef49e7b02c5b09f6dacbd1 + status: + code: 200 + message: OK +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Solve + this puzzle: call db_lookup(''start''), then follow each instruction step by + step. Each lookup tells you the next key to use. Compute each step in your head. + State only the final integer."},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfe62a9c8197914677ea9a2424a0","encrypted_content":"gAAAAABp-r_nDEFD_pLd6MEzoOCG8OKekq-Ev8vnbPTkSie4RovCRsX1uArYsm0yKKyAAEYh8tRzZVKUfeKd6HTYEqGrwUeI9Nd7K14uFr0VC8_AEFOX5_RmizhJInjfMLB5xr26SgIbSkBYpELGYzo4H-nlAJVwn_Tgni-Ka5vNjJZsh4-w5wUXQ3NaK4j0bXJYDwtKFlQ5lLzPa4KgzkWJk60HxK847Z5ZVk7WsH9SKDDPSLEsctmXxY5quJk9XUSGN5kR0TPKGscIxTXtzDPHcudCiAjdRGv0g_TIJlIUQ6ofDXWQl8bdpYoqQIbEy3Khqh3ksKoSG71n0UsUw-6d0IeBMQm04VcGiCUhDHaKa3EhFxh5MNgmuVEHDSp4ByweVwK7G4GniKd8NvkauFyROQ2Omqnj_6MKaWccIaQVm5Yw9WU0_PIPFDOKyse7cms2tlyGpFxeDgTgSARsxLaZEZMIWciZT1DazA9LrrD13aCD4ePHWCRJbpbkDH7FM3T2TrT16-G0kCip_OXm9FZZxJvAgUXMbnVEs1SugWBC2nTCfYrLRwXjyzZk35ekzv6LIy7Zigh0t-_RTOWrBkNxA-7jvgCP_YF51cG1no4hTFEHhhcuwYSmMSMMAkcWi0Dh-FXby7k9tjUGMKTjs1RN9NI8tc2lSwmAEdhZdk_JpEnVlfpDEsjRFRIE6RfghF-SRn2gFAUfRczeHKplmtWM0eDKiaK6XKHTPQats0mz1B16alRJOEeFbwMPQA1Myz7atKb8Z5I_IrlGBh1Ho9buaY_RFfCpGemPcO9eRrGoZ7QLwSQH6pw1MrDusZ_pt4vKYC3qDu3rEZ_6OHwFYnRPVj_Qu4bt9f5arKPTKAp2hmEsYKCDvaWeBp2lgEOb66sv34qH6qs-ooCAOG7rmtjSLCVu8F0R11NiFciwvmlmECOwT9d6emgKjn9T4_2CBaUmo9rdEiNfq-cmhMG7a26jGazsIl9Iz-prux7hV8871iYlNDhn_Mr86kkCmknKL53tGb-29XcK-jTUnlLGrHmhnBl06Cf0f-cE7sT6D2tjacDeh9P6gtu0enBfPbuJH75oA240HiKU7gRBd-kYnw6hvhCMKUkjb88Uj51OFOxB0vDLnF4_6y8rswyWoPVeeuRMWHJto_ersRSiuWnzVjJAo4dK9MLfBKRhIth6WDBu6ReL2575QfzUYEFTGXmGPX33OVw0-mgm","summary":[]},{"type":"function_call","call_id":"call_I670mAzR6AYszdoqaI96qg0k","name":"db_lookup","arguments":"{\"key\": + \"start\"}"},{"type":"function_call_output","call_id":"call_I670mAzR6AYszdoqaI96qg0k","output":"Begin + with the value 7."},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfe83488819788888aaafb2ab1fd","encrypted_content":"gAAAAABp-r_qaKH8G-A088KPm_AKkVfkzrE_w9jGidMqLh3fXoaw9IbtAS9DwhU_OCnhgjF9tD3K66xmrV4gVvDX5mfjjsF4A8rr9sdYPtMw1TmcwMntBVKvJskKjpjZe6s5dswnzCzuUzuG6UOKpRpoZpnmCIew5Sm-ZoxY0PjtXYj7_YdzkzXmb8M1nbFyymgIloaDarelYr71F-wysy_BIxPAfFulc7CeT_jGY6qTj-XaXEjNevpAkRxQs3qDFsf1vJJJSMRUu5vHFO-kmQUvC1C7nNSkK-AQxF4ghx2I_cGJNSHusnjGq5Ia397H_0CBOQjTd1HJIu3UWG8guMSjH9m2elLMFUpfExvbR4iK-l-ewCcWpIyfWxdLWrD3IWqmQ5Id8nEIJFK78mj4VmLPw8JOPkArncScgXqTK6uzkzEX2HtsNECxFR5w1eVZS1yixE6OSj4V5z-DDlxxx0ToibA1J9qcs5QKR-0ULx_LK_Pgf9AeK8_Z2FqHNnFh5nKVWdyIYXxTpurpF1qEnKBCSCwaQ7XzUJlJ-n1UFgZt0URFbsbqY3XshC4urrvqswPq1PLBqhQcZxPbsXqMKzFuwT6J6CEl2GrUoCjHbVDqIh1j6lc5BDYi7aOAYn7bEwQI_thfmQbi7EraVZMmgbEnvYkxvzeP78ASC4YiYwgIzGuUQKzwqRKKbxpe6oeVE79FCeXhEbKT2Z0kcBZSUomxfSCiLtNMC0Ni9WFfUOu94lRtTPdPH8pVu-QBFezi8GfxOXHNqNuFSVI_VQtenIoP4ia1eYxbI2hpwAlk3u8dUSCvJySdLLw9dPWNQxco51zDamyQVKenbnEbj4ZqZRKl-2t_neG0AcX5qliCqDA4mqmL74yirrmYt69kIFQ6SGCP-86km3amgAInOsOBVSQK--HaOvBUfTc-CWLOONqfj2aXUiMCKvSMvo9-6Fug4CsIXdcgxqOQYGfc5Opce8mTUdrSjmhb1WiMO6lk0LSrHq0IUsIh8Dn9X5bhXshn8dC5wl7Ya7Bofyg2EBGTo4Sc7i-nf2BfFX7kqTmOURGa4iZe8hNpuoytgVDlbjylf0NyU-GE0gwubFgmb6Dqn7UdJedE_i4DKIctupippecmwpgO9V1nilTlYoNYjrGryjTRZ6AEj8oaeR2R3G41wUpMHv6uUyYTIKl1l2cAd6cG0g5A8ECg9bQfIUcAAcQN3EdfoIoOXOEqCM-mCEjjP9wQcx4ztNg0_o_TT6CqZgPKDzzW85UgVDgtWGhU464SPEUoYxJrCpWixqQ_fTANdF4snP3hRG47yauIenFdO31zuafOJrcqEDycKflDHvA5YUOiK1iOoDK4abQhZBePmpWyOFf8lufsULGoetqiLG4VM5hdijIqkIOWidiKSBMxX8X75bAeT5uS3nCq6nS0jRLTKJ7LL9qH60med-9GfzMxKXnMHi0yewgVmQOgwir4rs8o2utjshbrOeDz4-Z-4pCskwZqdEqY0Q==","summary":[]},{"type":"function_call","call_id":"call_Uj506iEKjBZWrHPq2C3GDtgr","name":"db_lookup","arguments":"{\"key\": + \"7\"}"},{"type":"function_call_output","call_id":"call_Uj506iEKjBZWrHPq2C3GDtgr","output":"unknown + key"},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfeb569481978703af8553a073b9","encrypted_content":"gAAAAABp-r_vFGcSWQD083z5Jsq67CmK99_ZqqnwC8n6j4kVfWhNH5J9pJw5e7uV1QurddjHY1w7BdbV4HrfTkz4SUHio6aFeHMUrbQObDiRwP8kHX2JSY3m3wXaXABn5_qJi85Lj7CD6X73jzqdQpEeGQ-5zL3JsxO0k5UuZb01t5EuzuGzw_0LR1Tli0UJw43rKgCEqarHTus3l63d18pXuK5RMBR-3Z_xI93rKewxrkTaH1DBgxtlxtD5Z4f-d0-RbfG14Svq-QS3wsfg5Xrn85KJHwf8yJ55M752AxBArv_iqsV1YuKZAwyFxT0HiLrDOwI_G_QyYfD-0U-HsVGeESCS-g4jwDPDXst2aJPTOyhtTu0DbpXyU_Ukbrtlws4OOZurl-W7LvvXlWJNG0WWj6mpCcHrWMzubGkQSH87c-GXESHoBdxxReXGomDs4gqnbB7dYpgNRNveYzLnJRQdDYkdYHTJI7gBEVNFBw5wY0zXRhYtivh_G4NuL2FYcTPr8zFsxqIUXyLWJ2-95N0jvS_ap4R0H_l5YY8twKvT4stYA39T2PKt067Cl906VXoatJWjW9yfLjn1fIe2m3Wr4VXhkWUm7v7A5x5Q7UObihtAn-5NhoU9BK1FcMEY_0UmTclUAM-SFVrL3Dtf9GtoYoLH8PUb4Sdph9y0D6baVSNnBpydV18l9XtgruSH5p0t9cexYA3zzTAmMb9ujFxYAb_qWZ_BFSFnJo77lYCoiKWeNeEMl3YoW4tXqmosdpqlSFpHKrax-YGeMD1nagNqQFInSLegcESqI8l-Gyqws4X3VGxW_jPGJgjh_wZAWQTKMiKctcA-AP5JOpJxaPzZghRMGpdI73nOOTV_jQ9OVkKKWtjFeMuduFhWLtiichsWWSq8XwJDUKTRyY_LYBxzPFFxIhxkqQ27b_ndJPLvyDT9ATvN856mxSJYelK0uJjxqj13O5ZWICp0pdUGiELg00Q15UIBqllDQv71_Ltnk_oSEprXRlbifFswF8TNVFbvHh5ZF_ajeXNbSgMfs_qSt9HuZD2tybsSsYHqmitpar0udcgNKvyiPKPXab3kwiPYIj_7nvclcqCZUUzddMy5dA18jszEcY5N46eUV_NtZGO7hQ1Z5hSWrmPSeTUHFKg82QfmW8_ejb0y9qfPoxQcTf8XzVd7gaNDkjSoMC-5Fvbz__geDpkRbNqSPs9eFE5RK7TqkcB7PN2qNN-m7jnxgTaMf5zucA0Xj2SJjoTcDCNS22d_qikzo-jzOuiJD7SZzfST4ugJlf5bMUGIiOItD4e_jszixhZHWE8GHHdjFnplasuI2TUwjUSGvgWr66ELWmpwQaDya6bU2t_hrHQWhkiatXJYzizyI3lUGzf-6N1_ieunEEquLrpVad4LDLnYGI5obFQUJabrD-eQI-JfKRKGSlRBtgXCXtiTJg744kJv5VJsgcKyk4xjoeyAwWAL1mw0NKt0XOeJLdWGIlo7_WrVpFNtSngYnkAD7yXyWsoPbcpBpk_j32YT5aZ6NTeyT5jH_XDcu-pq6kF9sIdP6Ry0qSG0XEyz0zXpmUbL0vowzyNWPoKXdsWYVLXAc4IF-E_Ar_mzsMJkB_3Qalb2sT2dmW7UkaXO_Ej2PpA9aNW-qcV4q5st81qbQnW_JHvai0WzKEFimpEf70oa88Z7ARexPicizAGu-1xGTCZM8GqdtwurLiGOn8Wn9ojASn7zUEF_xTw9UcjB94jQwc_PKHY6BPBleKKPYTZWxAJwzLt937JwezczGHKfEDdM_DMVDI_vJ6H-mJ5nu8hAJK6UemD-GE4VSeSP_NtGZXXinevdm2k8HN7-KH6-zhwP8yMzw4NBnwCCdxvuxkTdhtp-69lY88c-_m6xyvkrpDkUDtsQDk6OjBYqe6b94tKnQh9I9FoVOP1-p4wNJNDxK2AjCTsEZIzD-g1ccSL2ZDtN9CmsXAfecl4RKzdyJgvHXbfFz1BRFaudrlk79C5khqva-WLrFfxkl6y9UE-12CFPt779pulpoeSXbR6PW0Y_mZXpFvtPH6N4Sryi3iYObDtxzgdhd6CNdrXovQstWULzuBS76EYyFPQ=","summary":[]},{"type":"function_call","call_id":"call_Vik9mkNli8KsnSbHOShapdmG","name":"db_lookup","arguments":"{\"key\": + \"step1\"}"},{"type":"function_call_output","call_id":"call_Vik9mkNli8KsnSbHOShapdmG","output":"unknown + key"}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"db_lookup","description":"Look + up a value by key in the puzzle database.","parameters":{"properties":{"key":{"type":"string"}},"required":["key"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '6407' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/7VXWY+byhJ+z6+w/BxH7MuR7oONd7BjG29wEqGmafbN7Caa/34BZ2xPltHVlWYe + RqaKqq7+auGrH596vb5j9P/p9ROUxhpGETzETWjSUDcBSWM4w5tAN5HJMDjG4TwLOMqkSBIwDNJx + CsL+59ZFpLsIZq9uojBFNzlMEMiQoYFWh7Msh1GNF7rTpRnI8rS1gVEQ+6h572akA+hZSZSHbVwm + 8FN0Ezu+74RWI/vRPDaCGFxR0tobqEB+FDcPjeLldvCry1+OZjstSpKotQxz3+8EZoIuOQrhVYtR + CPzs2iixL1inc8JXZ5qBMuD46bOlE6ZZksPMaS79LA9ApUV5FueZlkUe+l2ZRZGvQeC/dRdEBvLb + O1lxNqC/0AMCI5gBRg0Isn9/IQHtcc9mt5Mayb8dNDeAHqlN30msybG3xPIkADrCCJo1MAbQ3XGd + k+wao1tqQRqFbQruqgay5Bq3MMMozFDYlYA1bP9G8SDRqmCIHSplk5W25lKYvwLbQBAuziWdNDmj + vXg1on0RwAk6bVezwep4Hg/jhLVGMuPO1dV0PoiTNXvNRTgKx5zmE9VptZueRkeQVIPSkfFJ8HW9 + s5ZRKa+Xu5QUrQ03xwJ5dZlqijG18+mR2YOpcbamJm2iyW5EoulyI2yDFbPEw+RgR5PhCtF4eE2C + mObkmosJcWhuZ6Gqzom5V1nn0UUAZcYsM5uChSmnx42lXtyRGJKqlpbw6gWKEegbejY6wYXnRNHc + kJfqWYsDGRvUvspvKn2yGB/WsrQnYnG8Z3CqRvlFGcCcjmrdY1VKOCx5t9rUB+WcmwUQ1l6lMMyU + 5mCoZdh5W8acJx2yo30aDddidNnALbaXY++g6xeQz3OZzm3PmB3kwtIz63w4qfYQDGOg18l0NKOk + 6VlCcyxW1qeKitNks6cPXLCeHpWcXR4UHJVHiXd1e5f6Kn1ZDeURWU0p5utYwdwolFYckgZT+QLx + C1Geh5qxGe1ktNZ4MjxuvWNQm2LuKIEYFVitnA6B6IhSgfnzen0dOrCYhYUzPJMGPpsnTq5FohCv + dbkSJNbRgOBV+m6QoEKZzpOgWFUlT9hcAt1FMtsdancoT7jscNgPHaygLz7NHufBaO4UFEXklOib + 9jkWJ/MQSu5xuCk346OtZzhb4yJuX/e26ij78AICT+FOpDjYLCR5qs2AbJJHfOCWy9FmNxLgboFd + pXx+srV9GlkupSNF59fFhrC8oqykpWdNdqY0XM3MAbnFF6sa5ppURO5lRwrEAZKSRM9qM9Uoa2Kp + B9/kLtXBsOcInKpgyw3m/rYQ9v7MHawLd5IzYYIUcgJlLJZ0Fk0FPjB4b7rNFJUe1y7ciOaZZvF0 + eD3EWlAHy8lkKm3nllidwvJUjq76/CCpZooBIO7H2KR2YP7VqFUPKSZeHQTXdD3JH9i7VVktzDW1 + UIxtUMbCUZgGPlosBmKd5Bk/3qmhBxxv5wzT83aO02pQUu5xcXHxGb8/sV45qqvYTN1MbSbC5Dw4 + uXt/d6UENr5iuzGbqNnpxEjjNVAxvlj4yn8eYyHNgwAk7RD993sne/n8p7lkwr/OJROnWZ5r5xJP + cibOAQ4ShgkgwH+fS2YedvO3m6ZPQfzl+9IpQWLlQTOvOv2Pb30PXb/1//nW15HlhN/6L483W6fa + Ld7u50k/lWJJLZnJecvKIU+aVpCriHxYhCDo4jJ0zY8iL4/7/z8GALYY6BgweArAVtx8PumPxaAA + fo7exQBlpJ2uIe7ErmDmo6sfDMuLN7l+DAY61WEAyLYMaBLqBKnDD8Ygs1Gvw6HHvovEkkrijYxt + JzUlLIz0OBMn4ozxPqga9K4aeJpjSZMmWd6EkGbMj0Vi1HZEr3Qyu/cEypd3UbEMAioKg4veQpCt + EXNZG6wqMMzHoAK7+uChTnK0qeuESeE8gX0sKo1RkhHvghAuQnEYGOpXXYkvkPy6I09CA8n/UBrN + /+8drYtB0nhC/luy2PDNGyOOG7LdMDD0B9LaqAonylPtlY/fIrsTxjhp7po1LqGNtOZSf9UlqKV1 + N7rZJyj7xkIfVPDOxhsmGSUd/7Mdy/55yacvQev+Ts9TYKLs2sTU+jYd9IaMpygpnOZWmfNK702Q + +1n/59YQJejNXpChIG4pcd7J8Z8AZKjKHsE1oQXg8fxUFN17z7XXL1CiR6nTwdkPkOHkwWOvuCXC + jpr4WjXIs6h/V6S/E/Bfa++RfAOlMHHiV2ilpgB6edwDPztMv/aatPSa1mu7Ls7r2kc9A2RAByn6 + 8l4N3VVt9QRN+pL06eK3/DaANei+lTeaWyE8i56u0Ow5bcKflC/33y+fH+7bVcpJkHEH48n5/fn7 + k8HrAT+XyCcNMAynBQj4m+eYu9x/+uXkLr5uB23745dGyqK4gcdqbq63DrC7MO56hudugqTJ0utu + 1TecFOj+61Kap8BCj4JywjebHU0Sn39XPO2LPx5Toukq42GJvam9XzdGHOf/pPmT43tLPlmzb5xn + UQb8h5ah8XtZ5+nbFmzKBrTF1h7w8unlvzbc0z8nEAAA + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f752737fd3c0044-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 04:13:37 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '2162' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=_EXrqe7IjNWn6uSESYNRV6be_iYv5vfdiSH8V3vdNgs-1778040815.3529754-1.0.1.1-qv4Kk81buc06G3o0AUYL254nHY_tKoM.r4ToKt1atzCBnU7vDOrMSgZt87H9a__SsX0twv_Dqbwnv35sedBqkSVJ5tcTI3rPQI4rOXZ2o7bFUr43uDG7zWB_uR4Rv0xw; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 04:43:37 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_83b594dde5454daa8a186b3a26e8df84 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index bbaa94c4f..564d8e1fe 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -270,3 +270,108 @@ def can_have_dragons(population: int) -> bool: == pm["openai"]["encrypted_content"] ) assert reasoning_inputs[0]["id"] == pm["openai"]["id"] + + +@pytest.mark.vcr +def test_responses_interleaved_reasoning_between_tool_calls(vcr): + """Tool calls during reasoning: each turn produces fresh reasoning AND + every prior reasoning block is round-tripped on every subsequent turn + so the model's hidden chain of thought accumulates across the whole + chain. This is the GPT-5-class capability that the Chat Completions + API can't deliver because it discards reasoning between turns.""" + from llm.parts import ReasoningPart + + model = llm.get_model("gpt-5.5") + + # Tool whose results force the model to re-plan between calls: each + # lookup hands the model a NEW key to use next, so the model has to + # think to figure out the next argument. Parallel tool calls would + # short-circuit this, so we need the model to reason in series. + def db_lookup(key: str) -> str: + "Look up a value by key in the puzzle database." + table = { + "start": "Begin with the value 7.", + "step1_7": "Multiply by 13. Now lookup with key step2_.", + "step2_91": "Subtract 11. Now lookup with key step3_.", + "step3_80": ( + "The answer is the value modulo 9. State only the integer." + ), + } + return table.get(key, "unknown key") + + conversation = model.conversation(tools=[db_lookup]) + conversation.chain_limit = 4 + chain = conversation.chain( + "Solve this puzzle: call db_lookup('start'), then follow each " + "instruction step by step. Each lookup tells you the next key " + "to use. Compute each step in your head. State only the final " + "integer.", + stream=False, + options={"reasoning_effort": "high"}, + key=API_KEY, + ) + # The chain may exceed the limit - we just want enough turns to + # observe interleaved reasoning, then we stop. + try: + chain.text() + except ValueError as e: + if "Chain limit" not in str(e): + raise + + responses = chain._responses + assert len(responses) >= 3, ( + f"expected at least 3 chained turns, got {len(responses)}" + ) + + # 1) Fresh reasoning happens on more than just the first turn. This is + # the actual interleaved-reasoning capability, not just round-trip. + reasoning_token_counts = [] + for r in responses: + u = r.usage() + details = (u.details if u else None) or {} + reasoning_token_counts.append( + (details.get("output_tokens_details") or {}).get("reasoning_tokens") + or 0 + ) + turns_with_fresh_reasoning = sum(1 for n in reasoning_token_counts if n > 0) + assert turns_with_fresh_reasoning >= 2, ( + f"expected >=2 turns to produce fresh reasoning, got " + f"{turns_with_fresh_reasoning} (counts: {reasoning_token_counts})" + ) + + # 2) Every reasoning block produced earlier in the chain is round- + # tripped on every subsequent turn. The Nth turn's outgoing input + # must contain at least N-1 reasoning items. + for i in range(1, len(responses)): + outgoing = (responses[i]._prompt_json or {}).get("input") or [] + reasoning_count = sum( + 1 for it in outgoing if it.get("type") == "reasoning" + ) + # encrypted_content + id are non-empty on each one + for it in outgoing: + if it.get("type") == "reasoning": + assert it.get("encrypted_content"), "encrypted_content lost" + assert it.get("id"), "reasoning id lost" + assert reasoning_count >= i, ( + f"turn {i} must echo >= {i} reasoning items, got {reasoning_count}" + ) + + # 3) The captured ReasoningParts on the assistant messages carry the + # opaque metadata that was actually echoed back on the wire. + for i, r in enumerate(responses[:-1]): + rparts = [ + p + for m in r.messages() + for p in m.parts + if isinstance(p, ReasoningPart) + ] + if reasoning_token_counts[i] > 0: + assert rparts, ( + f"turn {i} produced reasoning_tokens={reasoning_token_counts[i]} " + "but no ReasoningPart was persisted" + ) + for rp in rparts: + pm = (rp.provider_metadata or {}).get("openai") or {} + assert pm.get("encrypted_content"), ( + "ReasoningPart missing encrypted_content" + ) From c564fbe24be218c1aeaf2325655fc8bdd070ab76 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 6 May 2026 04:48:51 +0000 Subject: [PATCH 076/258] Ran cog --- docs/openai-models.md | 4 ++-- docs/usage.md | 37 +++++++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/docs/openai-models.md b/docs/openai-models.md index d77f0936a..64c1a9ab9 100644 --- a/docs/openai-models.md +++ b/docs/openai-models.md @@ -75,8 +75,8 @@ OpenAI Chat: gpt-5.4-mini OpenAI Chat: gpt-5.4-mini-2026-03-17 OpenAI Chat: gpt-5.4-nano OpenAI Chat: gpt-5.4-nano-2026-03-17 -OpenAI Chat: gpt-5.5 -OpenAI Chat: gpt-5.5-2026-04-23 +OpenAI Responses: gpt-5.5 +OpenAI Responses: gpt-5.5-2026-04-23 OpenAI Completion: gpt-3.5-turbo-instruct (aliases: 3.5-instruct, chatgpt-instruct) ``` diff --git a/docs/usage.md b/docs/usage.md index e6f78a968..08ac2d22b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1489,20 +1489,52 @@ OpenAI Chat: gpt-5.4-nano-2026-03-17 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5.5 +OpenAI Responses: gpt-5.5 Options: temperature: float + What sampling temperature to use, between 0 and 2. Higher values like + 0.8 will make the output more random, while lower values like 0.2 will + make it more focused and deterministic. max_tokens: int + Maximum number of tokens to generate. top_p: float + An alternative to sampling with temperature, called nucleus sampling, + where the model considers the results of the tokens with top_p + probability mass. So 0.1 means only the tokens comprising the top 10% + probability mass are considered. Recommended to use top_p or + temperature but not both. frequency_penalty: float + Number between -2.0 and 2.0. Positive values penalize new tokens based + on their existing frequency in the text so far, decreasing the model's + likelihood to repeat the same line verbatim. presence_penalty: float + Number between -2.0 and 2.0. Positive values penalize new tokens based + on whether they appear in the text so far, increasing the model's + likelihood to talk about new topics. stop: str + A string where the API will stop generating further tokens. logit_bias: dict, str + Modify the likelihood of specified tokens appearing in the completion. + Pass a JSON string like '{"1712":-100, "892":-100, "1489":-100}' seed: int + Integer seed to attempt to sample deterministically json_object: boolean + Output a valid JSON object {...}. Prompt must mention JSON. + chat_completions: boolean + Force the use of the older /v1/chat/completions endpoint instead of + /v1/responses. Most callers should leave this off; set to true to fall + back to the Chat Completions code path for compatibility. image_detail: str + Controls the detail level for image attachments. Supported values are + low, high, original, and auto. reasoning_effort: str + Constraints effort on reasoning for reasoning models. Currently + supported values are low, medium, and high. Reducing reasoning effort + can result in faster responses and fewer tokens used on reasoning in a + response. verbosity: str + Controls how verbose the model's response should be. Supported values + are low, medium, and high. Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1513,7 +1545,7 @@ OpenAI Chat: gpt-5.5 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5.5-2026-04-23 +OpenAI Responses: gpt-5.5-2026-04-23 Options: temperature: float max_tokens: int @@ -1524,6 +1556,7 @@ OpenAI Chat: gpt-5.5-2026-04-23 logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str From 6952ff1c95cb3f949b4c0ad78cdb02ae09515896 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 May 2026 20:42:48 -0700 Subject: [PATCH 077/258] Ensure add_tool_call() is emitted as a Part, if necessary Response keeps two parallel stores: _stream_events (read by to_dict / response.messages) and _tool_calls (read by execute_tool_calls). Only the messages were correctly serialized, leaving _tool_calls unrecorded. Part assembly now also walks _tool_calls and appends a ToolCallPart for any tool_call_id not already represented by a StreamEvent-derived Part. Closes #1433 --- llm/models.py | 19 ++++++++++++ tests/test_parts.py | 73 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/llm/models.py b/llm/models.py index 20f044338..b929e7dc9 100644 --- a/llm/models.py +++ b/llm/models.py @@ -1154,6 +1154,25 @@ def _build_parts(self) -> List[Any]: ) ) + # Merge in any tool calls registered via add_tool_call() that the + # plugin didn't also emit as StreamEvents. Dedup by tool_call_id so + # plugins using both APIs in tandem don't double-count. + seen_ids = { + p.tool_call_id + for p in parts + if isinstance(p, ToolCallPart) and p.tool_call_id is not None + } + for tc in self._tool_calls: + if tc.tool_call_id is not None and tc.tool_call_id in seen_ids: + continue + parts.append( + ToolCallPart( + name=tc.name, + arguments=tc.arguments or {}, + tool_call_id=tc.tool_call_id, + ) + ) + # Hoist redacted reasoning Parts to the start of the assembled # message. Plugins typically emit them late (when usage arrives # in the final chunk), but UIs render reasoning before content, diff --git a/tests/test_parts.py b/tests/test_parts.py index 44687d75d..1fe359282 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -1225,6 +1225,79 @@ def tick() -> str: assert r3.prompt.messages[2].parts[0].tool_call_id == "c1" +class TestAddToolCallWithStreamEvents: + """A plugin may yield StreamEvents *and* call response.add_tool_call(). + The Part list must include the tool call regardless of whether + _stream_events is empty or populated; otherwise persistence drops the + tool call and the next turn ships an orphan tool_result. + """ + + def test_text_yield_plus_add_tool_call_emits_both_parts(self, mock_model): + class TextAndAddToolCallMock(type(mock_model)): + def execute(self, prompt, stream, response, conversation): + yield "answer" + response.add_tool_call( + llm.ToolCall( + name="search", + arguments={"q": "weather"}, + tool_call_id="c1", + ) + ) + + m = TextAndAddToolCallMock() + response = m.prompt("hi") + response.text() + parts = response.messages()[0].parts + assert llm.parts.TextPart(text="answer") in parts + tool_call_parts = [ + p for p in parts if isinstance(p, llm.parts.ToolCallPart) + ] + assert tool_call_parts == [ + llm.parts.ToolCallPart( + name="search", + arguments={"q": "weather"}, + tool_call_id="c1", + ) + ] + + def test_stream_event_tool_call_plus_matching_add_tool_call_dedups( + self, mock_model + ): + class DualApiMock(type(mock_model)): + def execute(self, prompt, stream, response, conversation): + yield llm.parts.StreamEvent( + type="tool_call_name", chunk="search", tool_call_id="c1" + ) + yield llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"q":"weather"}', + tool_call_id="c1", + ) + response.add_tool_call( + llm.ToolCall( + name="search", + arguments={"q": "weather"}, + tool_call_id="c1", + ) + ) + + m = DualApiMock() + response = m.prompt("hi") + response.text() + tool_call_parts = [ + p + for p in response.messages()[0].parts + if isinstance(p, llm.parts.ToolCallPart) + ] + assert tool_call_parts == [ + llm.parts.ToolCallPart( + name="search", + arguments={"q": "weather"}, + tool_call_id="c1", + ) + ] + + class TestResponseReply: def test_reply_builds_next_turn_from_this_response(self, mock_model): mock_model.enqueue(["a1"]) From 2838388c31b0c5e53cf3682af4b3164ad30a64c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 May 2026 21:18:40 -0700 Subject: [PATCH 078/258] Ran Black --- llm/default_plugins/openai_models.py | 32 +++++++---------- tests/test_openai_responses.py | 51 +++++++++++----------------- tests/test_parts.py | 4 +-- 3 files changed, 34 insertions(+), 53 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 2f2ca798a..bb42266fb 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1236,9 +1236,7 @@ def _build_responses_input(self, prompt, image_detail=None): for msg in prompt.messages: if msg.role == "system": - text = "".join( - p.text for p in msg.parts if isinstance(p, TextPart) - ) + text = "".join(p.text for p in msg.parts if isinstance(p, TextPart)) if text: instructions = text continue @@ -1569,9 +1567,7 @@ def execute( yield self._reasoning_event(item) elif item.type == "function_call": try: - args = ( - json.loads(item.arguments) if item.arguments else {} - ) + args = json.loads(item.arguments) if item.arguments else {} except json.JSONDecodeError: args = {"_raw": item.arguments} response.add_tool_call( @@ -1586,9 +1582,7 @@ def execute( if final_response_dict.get("usage"): usage = final_response_dict["usage"] if final_response_dict is not None: - response.response_json = remove_dict_none_values( - final_response_dict - ) + response.response_json = remove_dict_none_values(final_response_dict) else: completion = client.responses.create( model=self.model_name or self.model_id, @@ -1635,8 +1629,10 @@ def execute( # Fallback: usage said reasoning happened but the API gave us no # reasoning items to harvest encrypted_content from. Emit the # opaque "reasoning happened" marker for UI / token accounting. - if not had_reasoning and usage and ( - (usage.get("output_tokens_details") or {}).get("reasoning_tokens") + if ( + not had_reasoning + and usage + and ((usage.get("output_tokens_details") or {}).get("reasoning_tokens")) ): yield StreamEvent(type="reasoning", chunk="", redacted=True) response._prompt_json = redact_data( @@ -1774,9 +1770,7 @@ async def execute( yield self._reasoning_event(item) elif item.type == "function_call": try: - args = ( - json.loads(item.arguments) if item.arguments else {} - ) + args = json.loads(item.arguments) if item.arguments else {} except json.JSONDecodeError: args = {"_raw": item.arguments} response.add_tool_call( @@ -1791,9 +1785,7 @@ async def execute( if final_response_dict.get("usage"): usage = final_response_dict["usage"] if final_response_dict is not None: - response.response_json = remove_dict_none_values( - final_response_dict - ) + response.response_json = remove_dict_none_values(final_response_dict) else: completion = await client.responses.create( model=self.model_name or self.model_id, @@ -1837,8 +1829,10 @@ async def execute( yield StreamEvent(type="text", chunk=content.text) self._set_usage_responses(response, usage) - if not had_reasoning and usage and ( - (usage.get("output_tokens_details") or {}).get("reasoning_tokens") + if ( + not had_reasoning + and usage + and ((usage.get("output_tokens_details") or {}).get("reasoning_tokens")) ): yield StreamEvent(type="reasoning", chunk="", redacted=True) response._prompt_json = redact_data( diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index 564d8e1fe..052c6ccb9 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -58,7 +58,11 @@ def test_default_routes_to_responses_endpoint(httpx_mock): "role": "assistant", "status": "completed", "content": [ - {"type": "output_text", "text": "hi from responses", "annotations": []} + { + "type": "output_text", + "text": "hi from responses", + "annotations": [], + } ], } ], @@ -107,11 +111,7 @@ class FakePrompt: ), Message( role="tool", - parts=[ - ToolResultPart( - name="add", output="4", tool_call_id="call_abc" - ) - ], + parts=[ToolResultPart(name="add", output="4", tool_call_id="call_abc")], ), ] @@ -265,10 +265,7 @@ def can_have_dragons(population: int) -> bool: second_input = (second._prompt_json or {}).get("input") or [] reasoning_inputs = [it for it in second_input if it.get("type") == "reasoning"] assert reasoning_inputs, "second turn must echo a reasoning input item" - assert ( - reasoning_inputs[0]["encrypted_content"] - == pm["openai"]["encrypted_content"] - ) + assert reasoning_inputs[0]["encrypted_content"] == pm["openai"]["encrypted_content"] assert reasoning_inputs[0]["id"] == pm["openai"]["id"] @@ -293,9 +290,7 @@ def db_lookup(key: str) -> str: "start": "Begin with the value 7.", "step1_7": "Multiply by 13. Now lookup with key step2_.", "step2_91": "Subtract 11. Now lookup with key step3_.", - "step3_80": ( - "The answer is the value modulo 9. State only the integer." - ), + "step3_80": ("The answer is the value modulo 9. State only the integer."), } return table.get(key, "unknown key") @@ -319,9 +314,9 @@ def db_lookup(key: str) -> str: raise responses = chain._responses - assert len(responses) >= 3, ( - f"expected at least 3 chained turns, got {len(responses)}" - ) + assert ( + len(responses) >= 3 + ), f"expected at least 3 chained turns, got {len(responses)}" # 1) Fresh reasoning happens on more than just the first turn. This is # the actual interleaved-reasoning capability, not just round-trip. @@ -330,8 +325,7 @@ def db_lookup(key: str) -> str: u = r.usage() details = (u.details if u else None) or {} reasoning_token_counts.append( - (details.get("output_tokens_details") or {}).get("reasoning_tokens") - or 0 + (details.get("output_tokens_details") or {}).get("reasoning_tokens") or 0 ) turns_with_fresh_reasoning = sum(1 for n in reasoning_token_counts if n > 0) assert turns_with_fresh_reasoning >= 2, ( @@ -344,26 +338,21 @@ def db_lookup(key: str) -> str: # must contain at least N-1 reasoning items. for i in range(1, len(responses)): outgoing = (responses[i]._prompt_json or {}).get("input") or [] - reasoning_count = sum( - 1 for it in outgoing if it.get("type") == "reasoning" - ) + reasoning_count = sum(1 for it in outgoing if it.get("type") == "reasoning") # encrypted_content + id are non-empty on each one for it in outgoing: if it.get("type") == "reasoning": assert it.get("encrypted_content"), "encrypted_content lost" assert it.get("id"), "reasoning id lost" - assert reasoning_count >= i, ( - f"turn {i} must echo >= {i} reasoning items, got {reasoning_count}" - ) + assert ( + reasoning_count >= i + ), f"turn {i} must echo >= {i} reasoning items, got {reasoning_count}" # 3) The captured ReasoningParts on the assistant messages carry the # opaque metadata that was actually echoed back on the wire. for i, r in enumerate(responses[:-1]): rparts = [ - p - for m in r.messages() - for p in m.parts - if isinstance(p, ReasoningPart) + p for m in r.messages() for p in m.parts if isinstance(p, ReasoningPart) ] if reasoning_token_counts[i] > 0: assert rparts, ( @@ -372,6 +361,6 @@ def db_lookup(key: str) -> str: ) for rp in rparts: pm = (rp.provider_metadata or {}).get("openai") or {} - assert pm.get("encrypted_content"), ( - "ReasoningPart missing encrypted_content" - ) + assert pm.get( + "encrypted_content" + ), "ReasoningPart missing encrypted_content" diff --git a/tests/test_parts.py b/tests/test_parts.py index 1fe359282..f2a03e315 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -1249,9 +1249,7 @@ def execute(self, prompt, stream, response, conversation): response.text() parts = response.messages()[0].parts assert llm.parts.TextPart(text="answer") in parts - tool_call_parts = [ - p for p in parts if isinstance(p, llm.parts.ToolCallPart) - ] + tool_call_parts = [p for p in parts if isinstance(p, llm.parts.ToolCallPart)] assert tool_call_parts == [ llm.parts.ToolCallPart( name="search", From 2297a2aab060d67d2ed6b4b1062b0f51590a6704 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 May 2026 21:37:47 -0700 Subject: [PATCH 079/258] Fix for ruff --- tests/test_openai_responses.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index 052c6ccb9..73dcd9a3d 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -90,7 +90,6 @@ def test_responses_input_translation(): TextPart, ToolCallPart, ToolResultPart, - AttachmentPart, ) model = llm.get_model("gpt-5.5") From 98e651075f1e81303b1449f0944252f2e539c33b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 May 2026 22:33:28 -0700 Subject: [PATCH 080/258] Register more OpenAI models using Responses API --- llm/default_plugins/openai_models.py | 41 +++++---- tests/test_cli_openai_models.py | 127 +++++++++++++++++++++++++++ tests/test_openai_responses.py | 47 ++++++++++ 3 files changed, 197 insertions(+), 18 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index bb42266fb..84bb2aba9 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -124,7 +124,7 @@ def register_models(register): # o1 for model_id in ("o1", "o1-2024-12-17"): register( - Chat( + Responses( model_id, vision=True, can_stream=False, @@ -132,7 +132,7 @@ def register_models(register): supports_schema=True, supports_tools=True, ), - AsyncChat( + AsyncResponses( model_id, vision=True, can_stream=False, @@ -151,26 +151,26 @@ def register_models(register): AsyncChat("o1-mini", allows_system_prompt=False), ) register( - Chat("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), - AsyncChat("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), + Responses("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), + AsyncResponses("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), ) register( - Chat( + Responses( "o3", vision=True, reasoning=True, supports_schema=True, supports_tools=True ), - AsyncChat( + AsyncResponses( "o3", vision=True, reasoning=True, supports_schema=True, supports_tools=True ), ) register( - Chat( + Responses( "o4-mini", vision=True, reasoning=True, supports_schema=True, supports_tools=True, ), - AsyncChat( + AsyncResponses( "o4-mini", vision=True, reasoning=True, @@ -188,7 +188,7 @@ def register_models(register): "gpt-5-nano-2025-08-07", ): register( - Chat( + Responses( model_id, vision=True, reasoning=True, @@ -196,7 +196,7 @@ def register_models(register): supports_schema=True, supports_tools=True, ), - AsyncChat( + AsyncResponses( model_id, vision=True, reasoning=True, @@ -211,7 +211,7 @@ def register_models(register): "gpt-5.1-chat-latest", ): register( - Chat( + Responses( model_id, vision=True, reasoning=True, @@ -219,7 +219,7 @@ def register_models(register): supports_schema=True, supports_tools=True, ), - AsyncChat( + AsyncResponses( model_id, vision=True, reasoning=True, @@ -231,7 +231,7 @@ def register_models(register): # GPT-5.2 for model_id in ("gpt-5.2", "gpt-5.2-chat-latest"): register( - Chat( + Responses( model_id, vision=True, reasoning=True, @@ -239,7 +239,7 @@ def register_models(register): supports_schema=True, supports_tools=True, ), - AsyncChat( + AsyncResponses( model_id, vision=True, reasoning=True, @@ -260,7 +260,7 @@ def register_models(register): "gpt-5.4-nano-2026-03-17", ): register( - Chat( + Responses( model_id, vision=True, reasoning=True, @@ -269,7 +269,7 @@ def register_models(register): supports_schema=True, supports_tools=True, ), - AsyncChat( + AsyncResponses( model_id, vision=True, reasoning=True, @@ -342,6 +342,9 @@ def register_models(register): if extra_model.get("completion"): klass = Completion async_klass = None + elif extra_model.get("responses"): + klass = Responses + async_klass = AsyncResponses else: klass = Chat async_klass = AsyncChat @@ -1520,7 +1523,8 @@ def execute( if instructions is not None: kwargs["instructions"] = instructions kwargs["store"] = False - kwargs["include"] = ["reasoning.encrypted_content"] + if self._reasoning: + kwargs["include"] = ["reasoning.encrypted_content"] client = self.get_client(key) usage = None @@ -1723,7 +1727,8 @@ async def execute( if instructions is not None: kwargs["instructions"] = instructions kwargs["store"] = False - kwargs["include"] = ["reasoning.encrypted_content"] + if self._reasoning: + kwargs["include"] = ["reasoning.encrypted_content"] client = self.get_client(key, async_=True) usage = None diff --git a/tests/test_cli_openai_models.py b/tests/test_cli_openai_models.py index eabc7d84d..ac353eefb 100644 --- a/tests/test_cli_openai_models.py +++ b/tests/test_cli_openai_models.py @@ -101,6 +101,9 @@ def test_gpt5_verbosity_option_is_sent_to_openai_chat_completions(httpx_mock): "-m", "gpt-5", "-o", + "chat_completions", + "1", + "-o", "verbosity", "high", "--no-stream", @@ -116,6 +119,62 @@ def test_gpt5_verbosity_option_is_sent_to_openai_chat_completions(httpx_mock): assert "text" not in request_body +def test_gpt5_verbosity_option_is_sent_to_openai_responses_by_default(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_test_1", + "object": "response", + "created_at": 1, + "model": "gpt-5", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Verbose enough", + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + }, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "-m", + "gpt-5", + "-o", + "verbosity", + "high", + "--no-stream", + "--key", + "x", + "Say hi", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["text"]["verbosity"] == "high" + assert request_body["include"] == ["reasoning.encrypted_content"] + assert "verbosity" not in request_body + + def test_gpt5_verbosity_option_validates_allowed_values(): runner = CliRunner() result = runner.invoke( @@ -209,6 +268,9 @@ def test_openai_image_detail_original_is_sent_for_gpt54(httpx_mock): "-m", "gpt-5.4", "-o", + "chat_completions", + "1", + "-o", "image_detail", "original", "--at", @@ -227,6 +289,71 @@ def test_openai_image_detail_original_is_sent_for_gpt54(httpx_mock): assert image_part["image_url"]["detail"] == "original" +def test_openai_image_detail_original_is_sent_for_gpt54_responses_by_default( + httpx_mock, +): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_test_1", + "object": "response", + "created_at": 1, + "model": "gpt-5.4", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Original detail", + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + }, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "-m", + "gpt-5.4", + "-o", + "image_detail", + "original", + "--at", + "https://example.com/image.jpg", + "image/jpeg", + "--no-stream", + "--key", + "x", + "Describe this", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[-1].content) + image_part = request_body["input"][0]["content"][1] + assert image_part == { + "type": "input_image", + "image_url": "https://example.com/image.jpg", + "detail": "original", + } + assert "image_detail" not in request_body + + def test_openai_image_detail_original_is_rejected_for_other_models(): runner = CliRunner() result = runner.invoke( diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index 73dcd9a3d..bcde9732e 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -81,6 +81,53 @@ def test_default_routes_to_responses_endpoint(httpx_mock): # Ensure we sent to the right endpoint requests = [r for r in httpx_mock.get_requests()] assert any("/v1/responses" in str(r.url) for r in requests) + request_body = json.loads(requests[-1].content) + assert request_body["include"] == ["reasoning.encrypted_content"] + + +def test_non_reasoning_responses_model_omits_encrypted_reasoning_include(httpx_mock): + from llm.default_plugins.openai_models import Responses + + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_test_1", + "object": "response", + "created_at": 1, + "model": "gpt-4.1", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "hi from gpt-4.1", + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + }, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + + model = Responses("gpt-4.1", vision=True, supports_schema=True, supports_tools=True) + response = model.prompt("hello", stream=False, key="test") + + assert response.text() == "hi from gpt-4.1" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["model"] == "gpt-4.1" + assert "include" not in request_body def test_responses_input_translation(): From 7e6643d87de617722e69f95009d59ba043d10472 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 12 May 2026 05:34:11 +0000 Subject: [PATCH 081/258] Ran cog --- docs/openai-models.md | 42 +++++++-------- docs/usage.md | 123 ++++++++++++++++++++++++------------------ 2 files changed, 92 insertions(+), 73 deletions(-) diff --git a/docs/openai-models.md b/docs/openai-models.md index 64c1a9ab9..7cf9d5b32 100644 --- a/docs/openai-models.md +++ b/docs/openai-models.md @@ -52,29 +52,29 @@ OpenAI Chat: gpt-4-turbo-2024-04-09 OpenAI Chat: gpt-4-turbo (aliases: gpt-4-turbo-preview, 4-turbo, 4t) OpenAI Chat: gpt-4.5-preview-2025-02-27 OpenAI Chat: gpt-4.5-preview (aliases: gpt-4.5) -OpenAI Chat: o1 -OpenAI Chat: o1-2024-12-17 +OpenAI Responses: o1 +OpenAI Responses: o1-2024-12-17 OpenAI Chat: o1-preview OpenAI Chat: o1-mini -OpenAI Chat: o3-mini -OpenAI Chat: o3 -OpenAI Chat: o4-mini -OpenAI Chat: gpt-5 -OpenAI Chat: gpt-5-mini -OpenAI Chat: gpt-5-nano -OpenAI Chat: gpt-5-2025-08-07 -OpenAI Chat: gpt-5-mini-2025-08-07 -OpenAI Chat: gpt-5-nano-2025-08-07 -OpenAI Chat: gpt-5.1 -OpenAI Chat: gpt-5.1-chat-latest -OpenAI Chat: gpt-5.2 -OpenAI Chat: gpt-5.2-chat-latest -OpenAI Chat: gpt-5.4 -OpenAI Chat: gpt-5.4-2026-03-05 -OpenAI Chat: gpt-5.4-mini -OpenAI Chat: gpt-5.4-mini-2026-03-17 -OpenAI Chat: gpt-5.4-nano -OpenAI Chat: gpt-5.4-nano-2026-03-17 +OpenAI Responses: o3-mini +OpenAI Responses: o3 +OpenAI Responses: o4-mini +OpenAI Responses: gpt-5 +OpenAI Responses: gpt-5-mini +OpenAI Responses: gpt-5-nano +OpenAI Responses: gpt-5-2025-08-07 +OpenAI Responses: gpt-5-mini-2025-08-07 +OpenAI Responses: gpt-5-nano-2025-08-07 +OpenAI Responses: gpt-5.1 +OpenAI Responses: gpt-5.1-chat-latest +OpenAI Responses: gpt-5.2 +OpenAI Responses: gpt-5.2-chat-latest +OpenAI Responses: gpt-5.4 +OpenAI Responses: gpt-5.4-2026-03-05 +OpenAI Responses: gpt-5.4-mini +OpenAI Responses: gpt-5.4-mini-2026-03-17 +OpenAI Responses: gpt-5.4-nano +OpenAI Responses: gpt-5.4-nano-2026-03-17 OpenAI Responses: gpt-5.5 OpenAI Responses: gpt-5.5-2026-04-23 OpenAI Completion: gpt-3.5-turbo-instruct (aliases: 3.5-instruct, chatgpt-instruct) diff --git a/docs/usage.md b/docs/usage.md index 08ac2d22b..003418c8d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -958,19 +958,49 @@ OpenAI Chat: gpt-4.5-preview (aliases: gpt-4.5) Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: o1 +OpenAI Responses: o1 Options: temperature: float + What sampling temperature to use, between 0 and 2. Higher values like + 0.8 will make the output more random, while lower values like 0.2 will + make it more focused and deterministic. max_tokens: int + Maximum number of tokens to generate. top_p: float + An alternative to sampling with temperature, called nucleus sampling, + where the model considers the results of the tokens with top_p + probability mass. So 0.1 means only the tokens comprising the top 10% + probability mass are considered. Recommended to use top_p or + temperature but not both. frequency_penalty: float + Number between -2.0 and 2.0. Positive values penalize new tokens based + on their existing frequency in the text so far, decreasing the model's + likelihood to repeat the same line verbatim. presence_penalty: float + Number between -2.0 and 2.0. Positive values penalize new tokens based + on whether they appear in the text so far, increasing the model's + likelihood to talk about new topics. stop: str + A string where the API will stop generating further tokens. logit_bias: dict, str + Modify the likelihood of specified tokens appearing in the completion. + Pass a JSON string like '{"1712":-100, "892":-100, "1489":-100}' seed: int + Integer seed to attempt to sample deterministically json_object: boolean + Output a valid JSON object {...}. Prompt must mention JSON. + chat_completions: boolean + Force the use of the older /v1/chat/completions endpoint instead of + /v1/responses. Most callers should leave this off; set to true to fall + back to the Chat Completions code path for compatibility. image_detail: str + Controls the detail level for image attachments. Supported values are + low, high, and auto. reasoning_effort: str + Constraints effort on reasoning for reasoning models. Currently + supported values are low, medium, and high. Reducing reasoning effort + can result in faster responses and fewer tokens used on reasoning in a + response. Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -980,7 +1010,7 @@ OpenAI Chat: o1 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: o1-2024-12-17 +OpenAI Responses: o1-2024-12-17 Options: temperature: float max_tokens: int @@ -991,6 +1021,7 @@ OpenAI Chat: o1-2024-12-17 logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str Attachment types: @@ -1038,7 +1069,7 @@ OpenAI Chat: o1-mini Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: o3-mini +OpenAI Responses: o3-mini Options: temperature: float max_tokens: int @@ -1049,6 +1080,7 @@ OpenAI Chat: o3-mini logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str Features: @@ -1059,7 +1091,7 @@ OpenAI Chat: o3-mini Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: o3 +OpenAI Responses: o3 Options: temperature: float max_tokens: int @@ -1070,6 +1102,7 @@ OpenAI Chat: o3 logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str Attachment types: @@ -1082,7 +1115,7 @@ OpenAI Chat: o3 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: o4-mini +OpenAI Responses: o4-mini Options: temperature: float max_tokens: int @@ -1093,6 +1126,7 @@ OpenAI Chat: o4-mini logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str Attachment types: @@ -1105,7 +1139,7 @@ OpenAI Chat: o4-mini Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5 +OpenAI Responses: gpt-5 Options: temperature: float max_tokens: int @@ -1116,6 +1150,7 @@ OpenAI Chat: gpt-5 logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1129,7 +1164,7 @@ OpenAI Chat: gpt-5 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5-mini +OpenAI Responses: gpt-5-mini Options: temperature: float max_tokens: int @@ -1140,6 +1175,7 @@ OpenAI Chat: gpt-5-mini logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1153,7 +1189,7 @@ OpenAI Chat: gpt-5-mini Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5-nano +OpenAI Responses: gpt-5-nano Options: temperature: float max_tokens: int @@ -1164,6 +1200,7 @@ OpenAI Chat: gpt-5-nano logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1177,7 +1214,7 @@ OpenAI Chat: gpt-5-nano Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5-2025-08-07 +OpenAI Responses: gpt-5-2025-08-07 Options: temperature: float max_tokens: int @@ -1188,6 +1225,7 @@ OpenAI Chat: gpt-5-2025-08-07 logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1201,7 +1239,7 @@ OpenAI Chat: gpt-5-2025-08-07 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5-mini-2025-08-07 +OpenAI Responses: gpt-5-mini-2025-08-07 Options: temperature: float max_tokens: int @@ -1212,6 +1250,7 @@ OpenAI Chat: gpt-5-mini-2025-08-07 logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1225,7 +1264,7 @@ OpenAI Chat: gpt-5-mini-2025-08-07 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5-nano-2025-08-07 +OpenAI Responses: gpt-5-nano-2025-08-07 Options: temperature: float max_tokens: int @@ -1236,6 +1275,7 @@ OpenAI Chat: gpt-5-nano-2025-08-07 logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1249,7 +1289,7 @@ OpenAI Chat: gpt-5-nano-2025-08-07 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5.1 +OpenAI Responses: gpt-5.1 Options: temperature: float max_tokens: int @@ -1260,6 +1300,7 @@ OpenAI Chat: gpt-5.1 logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1273,7 +1314,7 @@ OpenAI Chat: gpt-5.1 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5.1-chat-latest +OpenAI Responses: gpt-5.1-chat-latest Options: temperature: float max_tokens: int @@ -1284,6 +1325,7 @@ OpenAI Chat: gpt-5.1-chat-latest logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1297,7 +1339,7 @@ OpenAI Chat: gpt-5.1-chat-latest Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5.2 +OpenAI Responses: gpt-5.2 Options: temperature: float max_tokens: int @@ -1308,6 +1350,7 @@ OpenAI Chat: gpt-5.2 logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1321,7 +1364,7 @@ OpenAI Chat: gpt-5.2 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5.2-chat-latest +OpenAI Responses: gpt-5.2-chat-latest Options: temperature: float max_tokens: int @@ -1332,6 +1375,7 @@ OpenAI Chat: gpt-5.2-chat-latest logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1345,7 +1389,7 @@ OpenAI Chat: gpt-5.2-chat-latest Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5.4 +OpenAI Responses: gpt-5.4 Options: temperature: float max_tokens: int @@ -1356,6 +1400,7 @@ OpenAI Chat: gpt-5.4 logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1369,7 +1414,7 @@ OpenAI Chat: gpt-5.4 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5.4-2026-03-05 +OpenAI Responses: gpt-5.4-2026-03-05 Options: temperature: float max_tokens: int @@ -1380,6 +1425,7 @@ OpenAI Chat: gpt-5.4-2026-03-05 logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1393,7 +1439,7 @@ OpenAI Chat: gpt-5.4-2026-03-05 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5.4-mini +OpenAI Responses: gpt-5.4-mini Options: temperature: float max_tokens: int @@ -1404,6 +1450,7 @@ OpenAI Chat: gpt-5.4-mini logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1417,7 +1464,7 @@ OpenAI Chat: gpt-5.4-mini Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5.4-mini-2026-03-17 +OpenAI Responses: gpt-5.4-mini-2026-03-17 Options: temperature: float max_tokens: int @@ -1428,6 +1475,7 @@ OpenAI Chat: gpt-5.4-mini-2026-03-17 logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1441,7 +1489,7 @@ OpenAI Chat: gpt-5.4-mini-2026-03-17 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5.4-nano +OpenAI Responses: gpt-5.4-nano Options: temperature: float max_tokens: int @@ -1452,6 +1500,7 @@ OpenAI Chat: gpt-5.4-nano logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1465,7 +1514,7 @@ OpenAI Chat: gpt-5.4-nano Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-5.4-nano-2026-03-17 +OpenAI Responses: gpt-5.4-nano-2026-03-17 Options: temperature: float max_tokens: int @@ -1476,6 +1525,7 @@ OpenAI Chat: gpt-5.4-nano-2026-03-17 logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str reasoning_effort: str verbosity: str @@ -1492,49 +1542,18 @@ OpenAI Chat: gpt-5.4-nano-2026-03-17 OpenAI Responses: gpt-5.5 Options: temperature: float - What sampling temperature to use, between 0 and 2. Higher values like - 0.8 will make the output more random, while lower values like 0.2 will - make it more focused and deterministic. max_tokens: int - Maximum number of tokens to generate. top_p: float - An alternative to sampling with temperature, called nucleus sampling, - where the model considers the results of the tokens with top_p - probability mass. So 0.1 means only the tokens comprising the top 10% - probability mass are considered. Recommended to use top_p or - temperature but not both. frequency_penalty: float - Number between -2.0 and 2.0. Positive values penalize new tokens based - on their existing frequency in the text so far, decreasing the model's - likelihood to repeat the same line verbatim. presence_penalty: float - Number between -2.0 and 2.0. Positive values penalize new tokens based - on whether they appear in the text so far, increasing the model's - likelihood to talk about new topics. stop: str - A string where the API will stop generating further tokens. logit_bias: dict, str - Modify the likelihood of specified tokens appearing in the completion. - Pass a JSON string like '{"1712":-100, "892":-100, "1489":-100}' seed: int - Integer seed to attempt to sample deterministically json_object: boolean - Output a valid JSON object {...}. Prompt must mention JSON. chat_completions: boolean - Force the use of the older /v1/chat/completions endpoint instead of - /v1/responses. Most callers should leave this off; set to true to fall - back to the Chat Completions code path for compatibility. image_detail: str - Controls the detail level for image attachments. Supported values are - low, high, original, and auto. reasoning_effort: str - Constraints effort on reasoning for reasoning models. Currently - supported values are low, medium, and high. Reducing reasoning effort - can result in faster responses and fewer tokens used on reasoning in a - response. verbosity: str - Controls how verbose the model's response should be. Supported values - are low, medium, and high. Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: From 3b5eaedb87ce7fc560a9548b3fbbdbb13e717803 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 May 2026 22:39:30 -0700 Subject: [PATCH 082/258] Run latest black --- llm/default_plugins/openai_models.py | 4 +++- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 84bb2aba9..1554b9c57 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -152,7 +152,9 @@ def register_models(register): ) register( Responses("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), - AsyncResponses("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), + AsyncResponses( + "o3-mini", reasoning=True, supports_schema=True, supports_tools=True + ), ) register( Responses( diff --git a/pyproject.toml b/pyproject.toml index 9ca5d3c22..948302833 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ dev = [ "pytest-asyncio", "cogapp", "mypy>=1.10.0", - "black>=25.1.0", + "black>=26.3.1", "pytest-recording", "ruff", "syrupy", From 3e9e3a30eab36a27a6cc972dc1d9f67b49c1aacf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 May 2026 23:06:52 -0700 Subject: [PATCH 083/258] Request reasoning summary auto for OpenAI models --- llm/default_plugins/openai_models.py | 82 ++++++++++++++++++-- tests/test_openai_responses.py | 110 ++++++++++++++++++++++++++- 2 files changed, 184 insertions(+), 8 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 1554b9c57..a21dd4127 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1358,8 +1358,11 @@ def _build_responses_kwargs(self, prompt, stream): kwargs["top_p"] = top_p if seed is not None: kwargs["seed"] = seed - if reasoning_effort: - kwargs["reasoning"] = {"effort": reasoning_effort} + if self._reasoning: + reasoning = {"summary": "auto"} + if reasoning_effort: + reasoning["effort"] = reasoning_effort + kwargs["reasoning"] = reasoning text: Dict[str, Any] = {} if verbosity: @@ -1410,7 +1413,19 @@ def _set_usage_responses(self, response, usage): input=input_tokens, output=output_tokens, details=details or None ) - def _reasoning_event(self, item): + def _reasoning_text_from_item(self, item): + bits = [] + for attr in ("summary", "content"): + for part in getattr(item, attr, None) or []: + if isinstance(part, dict): + text = part.get("text") + else: + text = getattr(part, "text", None) + if text: + bits.append(text) + return "".join(bits) + + def _reasoning_event(self, item, *, include_text=True): """Build a redacted-reasoning StreamEvent that carries the opaque ``id`` and ``encrypted_content`` from a Responses-API reasoning item. Echoing this metadata back on the next request via @@ -1420,6 +1435,7 @@ def _reasoning_event(self, item): rid = getattr(item, "id", None) enc = getattr(item, "encrypted_content", None) summary = getattr(item, "summary", None) + text = self._reasoning_text_from_item(item) if include_text else "" meta: Dict[str, Any] = {} if rid: meta["id"] = rid @@ -1437,8 +1453,8 @@ def _reasoning_event(self, item): meta["summary"] = list(summary) return StreamEvent( type="reasoning", - chunk="", - redacted=True, + chunk=text, + redacted=include_text and not text, provider_metadata={"openai": meta} if meta else None, ) @@ -1540,6 +1556,7 @@ def execute( ) tool_call_meta: Dict[str, Dict[str, str]] = {} final_response_dict: Optional[Dict[str, Any]] = None + reasoning_items_with_streamed_text = set() for event in stream_obj: etype = getattr(event, "type", None) if etype == "response.output_item.added": @@ -1566,11 +1583,36 @@ def execute( chunk=event.delta or "", tool_call_id=call_id, ) + elif etype in ( + "response.reasoning_summary_text.delta", + "response.reasoning_text.delta", + ): + item_id = getattr(event, "item_id", None) + if item_id: + reasoning_items_with_streamed_text.add(item_id) + yield StreamEvent(type="reasoning", chunk=event.delta or "") + elif etype in ( + "response.reasoning_summary_text.done", + "response.reasoning_text.done", + ): + item_id = getattr(event, "item_id", None) + if item_id not in reasoning_items_with_streamed_text: + text = getattr(event, "text", None) or "" + if text: + if item_id: + reasoning_items_with_streamed_text.add(item_id) + yield StreamEvent(type="reasoning", chunk=text) elif etype == "response.output_item.done": item = event.item if item.type == "reasoning": had_reasoning = True - yield self._reasoning_event(item) + item_id = getattr(item, "id", None) + yield self._reasoning_event( + item, + include_text=( + item_id not in reasoning_items_with_streamed_text + ), + ) elif item.type == "function_call": try: args = json.loads(item.arguments) if item.arguments else {} @@ -1744,6 +1786,7 @@ async def execute( ) tool_call_meta: Dict[str, Dict[str, str]] = {} final_response_dict: Optional[Dict[str, Any]] = None + reasoning_items_with_streamed_text = set() async for event in stream_obj: etype = getattr(event, "type", None) if etype == "response.output_item.added": @@ -1770,11 +1813,36 @@ async def execute( chunk=event.delta or "", tool_call_id=call_id, ) + elif etype in ( + "response.reasoning_summary_text.delta", + "response.reasoning_text.delta", + ): + item_id = getattr(event, "item_id", None) + if item_id: + reasoning_items_with_streamed_text.add(item_id) + yield StreamEvent(type="reasoning", chunk=event.delta or "") + elif etype in ( + "response.reasoning_summary_text.done", + "response.reasoning_text.done", + ): + item_id = getattr(event, "item_id", None) + if item_id not in reasoning_items_with_streamed_text: + text = getattr(event, "text", None) or "" + if text: + if item_id: + reasoning_items_with_streamed_text.add(item_id) + yield StreamEvent(type="reasoning", chunk=text) elif etype == "response.output_item.done": item = event.item if item.type == "reasoning": had_reasoning = True - yield self._reasoning_event(item) + item_id = getattr(item, "id", None) + yield self._reasoning_event( + item, + include_text=( + item_id not in reasoning_items_with_streamed_text + ), + ) elif item.type == "function_call": try: args = json.loads(item.arguments) if item.arguments else {} diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index bcde9732e..78536a1c3 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -5,10 +5,64 @@ import llm import pytest +from pytest_httpx import IteratorStream API_KEY = os.environ.get("PYTEST_OPENAI_API_KEY", None) or "badkey" +def _responses_sse(event_type, data): + data = {"type": event_type, **data} + return f"event: {event_type}\ndata: {json.dumps(data)}\n\n".encode("utf-8") + + +def _responses_reasoning_summary_stream(): + yield _responses_sse( + "response.reasoning_summary_text.delta", + { + "item_id": "rs_1", + "output_index": 0, + "summary_index": 0, + "delta": "Thinking", + "sequence_number": 1, + }, + ) + yield _responses_sse( + "response.reasoning_summary_text.delta", + { + "item_id": "rs_1", + "output_index": 0, + "summary_index": 0, + "delta": " aloud", + "sequence_number": 2, + }, + ) + yield _responses_sse( + "response.output_item.done", + { + "item": { + "id": "rs_1", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Thinking aloud"}], + "encrypted_content": "encrypted", + "status": "completed", + }, + "output_index": 0, + "sequence_number": 3, + }, + ) + yield _responses_sse( + "response.output_text.delta", + { + "item_id": "msg_1", + "output_index": 1, + "content_index": 0, + "delta": "done", + "logprobs": [], + "sequence_number": 4, + }, + ) + + def test_responses_model_is_registered(): model = llm.get_model("gpt-5.5") assert "Responses" in type(model).__name__ @@ -83,6 +137,7 @@ def test_default_routes_to_responses_endpoint(httpx_mock): assert any("/v1/responses" in str(r.url) for r in requests) request_body = json.loads(requests[-1].content) assert request_body["include"] == ["reasoning.encrypted_content"] + assert request_body["reasoning"] == {"summary": "auto"} def test_non_reasoning_responses_model_omits_encrypted_reasoning_include(httpx_mock): @@ -128,6 +183,7 @@ def test_non_reasoning_responses_model_omits_encrypted_reasoning_include(httpx_m request_body = json.loads(httpx_mock.get_requests()[-1].content) assert request_body["model"] == "gpt-4.1" assert "include" not in request_body + assert "reasoning" not in request_body def test_responses_input_translation(): @@ -190,10 +246,62 @@ class FakePrompt: p.tools = [] p.schema = None kwargs = model._build_responses_kwargs(p, stream=False) - assert kwargs["reasoning"] == {"effort": "low"} + assert kwargs["reasoning"] == {"summary": "auto", "effort": "low"} assert kwargs["text"]["verbosity"] == "low" +def test_responses_kwargs_sets_reasoning_summary_without_effort(): + model = llm.get_model("gpt-5.5") + options = model.Options() + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + kwargs = model._build_responses_kwargs(p, stream=False) + assert kwargs["reasoning"] == {"summary": "auto"} + + +def test_responses_streams_reasoning_summary_text(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + stream=IteratorStream(_responses_reasoning_summary_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + + model = llm.get_model("gpt-5.5") + response = model.prompt("hello", key="test") + events = list(response.stream_events()) + + assert [(e.type, e.chunk) for e in events] == [ + ("reasoning", "Thinking"), + ("reasoning", " aloud"), + ("reasoning", ""), + ("text", "done"), + ] + messages = response.messages() + reasoning_parts = [ + p for m in messages for p in m.parts if isinstance(p, llm.parts.ReasoningPart) + ] + assert reasoning_parts == [ + llm.parts.ReasoningPart( + text="Thinking aloud", + provider_metadata={ + "openai": { + "id": "rs_1", + "encrypted_content": "encrypted", + "summary": [{"type": "summary_text", "text": "Thinking aloud"}], + } + }, + ) + ] + assert response.text() == "done" + + @pytest.mark.vcr def test_responses_basic_non_streaming(vcr): model = llm.get_model("gpt-5.5") From 6e12258c0bd2e5ce0bfe3308fe745b50ce23b1cd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 May 2026 08:17:03 -0700 Subject: [PATCH 084/258] Send prior assistant text as plain string in OpenAI Responses input When building the `input` list for the OpenAI Responses API from prior conversation turns, an assistant text-only turn was being serialized as: {"role": "assistant", "content": [{"type": "output_text", "text": "..."}]} The openai-python SDK's EasyInputMessage shape uses a plain string for this case, matching what a direct OpenAI Responses call would send. Use the same shape so our history matches the SDK exactly, and add tests covering both _build_responses_input and a two-turn response.reply() flow. --- llm/default_plugins/openai_models.py | 12 +--- tests/test_openai_responses.py | 84 ++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 11 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index a21dd4127..09367c0de 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1317,17 +1317,7 @@ def _build_responses_input(self, prompt, image_detail=None): items.append({"role": "user", "content": "".join(text_bits)}) elif msg.role == "assistant": if text_bits: - items.append( - { - "role": "assistant", - "content": [ - { - "type": "output_text", - "text": "".join(text_bits), - } - ], - } - ) + items.append({"role": "assistant", "content": "".join(text_bits)}) items.extend(tool_call_items) return items, instructions diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index 78536a1c3..1527e0875 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -234,6 +234,90 @@ class FakePrompt: } +def test_responses_input_translation_assistant_text_uses_easy_input_message(): + """Plain prior assistant text should match OpenAI's EasyInputMessage shape.""" + from llm.parts import Message, TextPart + + model = llm.get_model("gpt-5.5") + + class FakePrompt: + messages = [ + Message(role="user", parts=[TextPart(text="hello")]), + Message(role="assistant", parts=[TextPart(text="first-ok")]), + Message(role="user", parts=[TextPart(text="what next?")]), + ] + + items, instructions = model._build_responses_input(FakePrompt()) + + assert instructions is None + assert items == [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "first-ok"}, + {"role": "user", "content": "what next?"}, + ] + + +def test_responses_reply_sends_prior_assistant_text_as_string(httpx_mock): + """response.reply() should send the same simple history shape a direct + openai-python Responses call would use for a text-only assistant turn.""" + + def response_json(response_id, message_id, text): + return { + "id": response_id, + "object": "response", + "created_at": 1, + "model": "gpt-5.5", + "output": [ + { + "type": "message", + "id": message_id, + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": text, + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + }, + "status": "completed", + } + + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json=response_json("resp_1", "msg_1", "first-ok"), + headers={"Content-Type": "application/json"}, + ) + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json=response_json("resp_2", "msg_2", "followup-ok"), + headers={"Content-Type": "application/json"}, + ) + + model = llm.get_model("gpt-5.5") + first = model.prompt("Say exactly: first-ok", stream=False, key="test") + second = first.reply("Say exactly: followup-ok", stream=False, key="test") + + assert first.text() == "first-ok" + assert second.text() == "followup-ok" + requests = httpx_mock.get_requests() + second_body = json.loads(requests[-1].content) + assert second_body["input"] == [ + {"role": "user", "content": "Say exactly: first-ok"}, + {"role": "assistant", "content": "first-ok"}, + {"role": "user", "content": "Say exactly: followup-ok"}, + ] + + def test_responses_kwargs_packs_reasoning_and_verbosity(): model = llm.get_model("gpt-5.5") options = model.Options(reasoning_effort="low", verbosity="low") From 74437d3dfec1020a74067bab06fef5c73af83c7a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 May 2026 08:38:47 -0700 Subject: [PATCH 085/258] "llm -m model --options" to see model options --- docs/help.md | 3 +- docs/usage.md | 2 +- llm/cli.py | 189 ++++++++++++++++++++++++++++++---------------- tests/test_llm.py | 15 ++++ 4 files changed, 140 insertions(+), 69 deletions(-) diff --git a/docs/help.md b/docs/help.md index fa774ba33..3ac4e317b 100644 --- a/docs/help.md +++ b/docs/help.md @@ -134,6 +134,7 @@ Options: --cl, --chain-limit INTEGER How many chained tool responses to allow, default 5, set 0 for unlimited -o, --option ... key/value options for the model + --options Show options for the selected model --schema TEXT JSON schema, filepath or ID --schema-multi TEXT JSON schema to use for multiple results -f, --fragment TEXT Fragment (alias, URL, hash or file path) to @@ -1084,4 +1085,4 @@ Options: --key TEXT OpenAI API key -h, --help Show this message and exit. ``` - \ No newline at end of file + diff --git a/docs/usage.md b/docs/usage.md index 003418c8d..1b237f75b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -59,7 +59,7 @@ Some models support options. You can pass these using `-o/--option name value` - llm 'Ten names for cheesecakes' -o temperature 1.5 ``` -Use the `llm models --options` command to see which options are supported by each model. +Use the `llm models --options` command to see which options are supported by each model, or `llm -m gpt-5.5 --options` to show the options for a specific selected model. You can also {ref}`configure default options ` for a model using the `llm models options` commands. diff --git a/llm/cli.py b/llm/cli.py index ead5d0a43..f33a7cc76 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -453,6 +453,12 @@ def cli(): multiple=True, help="key/value options for the model", ) +@click.option( + "show_model_options", + "--options", + is_flag=True, + help="Show options for the selected model", +) @schema_option @click.option( "--schema-multi", @@ -526,6 +532,7 @@ def prompt( tools_approve, chain_limit, options, + show_model_options, schema_input, schema_multi, fragments, @@ -575,11 +582,6 @@ def prompt( if log and no_log: raise click.ClickException("--log and --no-log are mutually exclusive") - log_path = pathlib.Path(database) if database else logs_db_path() - (log_path.parent).mkdir(parents=True, exist_ok=True) - db = sqlite_utils.Database(log_path) - migrate(db) - if queries and not model_id: # Use -q options to find model with shortest model_id matches = [] @@ -592,6 +594,23 @@ def prompt( ) model_id = min(matches, key=len) + if show_model_options and not (conversation_id or _continue or template): + model_id = model_id or get_default_model() + try: + if async_: + get_async_model(model_id) + else: + get_model(model_id) + except UnknownModelError as ex: + raise click.ClickException(ex) + click.echo(render_model_with_options(model_id, async_=async_)) + return + + log_path = pathlib.Path(database) if database else logs_db_path() + (log_path.parent).mkdir(parents=True, exist_ok=True) + db = sqlite_utils.Database(log_path) + migrate(db) + if schema_multi: schema_input = schema_multi @@ -795,6 +814,10 @@ def read_prompt(): except UnknownModelError as ex: raise click.ClickException(ex) + if show_model_options: + click.echo(render_model_with_options(model_id, async_=async_)) + return + if conversation is None and (tools or python_tools): conversation = model.conversation() @@ -2273,6 +2296,93 @@ def models(): } +def model_matches_id_or_alias(model_with_aliases, model_ids): + ids_and_aliases = set( + [model_with_aliases.model.model_id] + model_with_aliases.aliases + ) + return ids_and_aliases.intersection(model_ids) + + +def render_model_with_aliases( + model_with_aliases, + *, + options=False, + async_=False, + models_that_have_shown_options=None, +): + extra_info = [] + if model_with_aliases.aliases: + extra_info.append("aliases: {}".format(", ".join(model_with_aliases.aliases))) + model = model_with_aliases.model if not async_ else model_with_aliases.async_model + output = str(model) + if extra_info: + output += " ({})".format(", ".join(extra_info)) + if options and model.Options.model_json_schema()["properties"]: + output += "\n Options:" + for name, field in model.Options.model_json_schema()["properties"].items(): + any_of = field.get("anyOf") + if any_of is None: + any_of = [{"type": field.get("type", "str")}] + types = ", ".join( + [ + _type_lookup.get(item.get("type"), item.get("type", "str")) + for item in any_of + if item.get("type") != "null" + ] + ) + bits = ["\n ", name, ": ", types] + description = field.get("description", "") + if ( + description + and models_that_have_shown_options is not None + and model.__class__ not in models_that_have_shown_options + ): + wrapped = textwrap.wrap(description, 70) + bits.append("\n ") + bits.extend("\n ".join(wrapped)) + output += "".join(bits) + if models_that_have_shown_options is not None: + models_that_have_shown_options.add(model.__class__) + if options and model.attachment_types: + attachment_types = ", ".join(sorted(model.attachment_types)) + wrapper = textwrap.TextWrapper( + width=min(max(shutil.get_terminal_size().columns, 30), 70), + initial_indent=" ", + subsequent_indent=" ", + ) + output += "\n Attachment types:\n{}".format(wrapper.fill(attachment_types)) + features = ( + [] + + (["streaming"] if model.can_stream else []) + + (["schemas"] if model.supports_schema else []) + + (["tools"] if model.supports_tools else []) + + (["async"] if model_with_aliases.async_model else []) + ) + if options and features: + output += "\n Features:\n{}".format( + "\n".join(" - {}".format(feature) for feature in features) + ) + if options and hasattr(model, "needs_key") and model.needs_key: + output += "\n Keys:" + if hasattr(model, "needs_key") and model.needs_key: + output += "\n key: {}".format(model.needs_key) + if hasattr(model, "key_env_var") and model.key_env_var: + output += "\n env_var: {}".format(model.key_env_var) + return output + + +def render_model_with_options(model_id, *, async_=False): + for model_with_aliases in get_models_with_aliases(): + if model_matches_id_or_alias(model_with_aliases, [model_id]): + return render_model_with_aliases( + model_with_aliases, + options=True, + async_=async_, + models_that_have_shown_options=set(), + ) + raise click.ClickException("'{}' is not a known model".format(model_id)) + + @models.command(name="list") @click.option( "--options", is_flag=True, help="Show options for each model, if available" @@ -2298,75 +2408,20 @@ def models_list(options, async_, schemas, tools, query, model_ids): if not all(model_with_aliases.matches(q) for q in query): continue if model_ids: - ids_and_aliases = set( - [model_with_aliases.model.model_id] + model_with_aliases.aliases - ) - if not ids_and_aliases.intersection(model_ids): + if not model_matches_id_or_alias(model_with_aliases, model_ids): continue if schemas and not model_with_aliases.model.supports_schema: continue if tools and not model_with_aliases.model.supports_tools: continue - extra_info = [] - if model_with_aliases.aliases: - extra_info.append( - "aliases: {}".format(", ".join(model_with_aliases.aliases)) - ) - model = ( - model_with_aliases.model if not async_ else model_with_aliases.async_model - ) - output = str(model) - if extra_info: - output += " ({})".format(", ".join(extra_info)) - if options and model.Options.model_json_schema()["properties"]: - output += "\n Options:" - for name, field in model.Options.model_json_schema()["properties"].items(): - any_of = field.get("anyOf") - if any_of is None: - any_of = [{"type": field.get("type", "str")}] - types = ", ".join( - [ - _type_lookup.get(item.get("type"), item.get("type", "str")) - for item in any_of - if item.get("type") != "null" - ] - ) - bits = ["\n ", name, ": ", types] - description = field.get("description", "") - if description and ( - model.__class__ not in models_that_have_shown_options - ): - wrapped = textwrap.wrap(description, 70) - bits.append("\n ") - bits.extend("\n ".join(wrapped)) - output += "".join(bits) - models_that_have_shown_options.add(model.__class__) - if options and model.attachment_types: - attachment_types = ", ".join(sorted(model.attachment_types)) - wrapper = textwrap.TextWrapper( - width=min(max(shutil.get_terminal_size().columns, 30), 70), - initial_indent=" ", - subsequent_indent=" ", + click.echo( + render_model_with_aliases( + model_with_aliases, + options=options, + async_=async_, + models_that_have_shown_options=models_that_have_shown_options, ) - output += "\n Attachment types:\n{}".format(wrapper.fill(attachment_types)) - features = ( - [] - + (["streaming"] if model.can_stream else []) - + (["schemas"] if model.supports_schema else []) - + (["tools"] if model.supports_tools else []) - + (["async"] if model_with_aliases.async_model else []) ) - if options and features: - output += "\n Features:\n{}".format( - "\n".join(" - {}".format(feature) for feature in features) - ) - if options and hasattr(model, "needs_key") and model.needs_key: - output += "\n Keys:" - if hasattr(model, "needs_key") and model.needs_key: - output += "\n key: {}".format(model.needs_key) - if hasattr(model, "key_env_var") and model.key_env_var: - output += "\n env_var: {}".format(model.key_env_var) - click.echo(output) if not query and not options and not schemas and not model_ids: click.echo(f"Default: {get_default_model()}") diff --git a/tests/test_llm.py b/tests/test_llm.py index 526cc1258..bd98d070b 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -520,6 +520,21 @@ def test_llm_models_options(user_path): assert "AsyncMockModel (async): mock" not in result.output +def test_prompt_options_shows_selected_model_options(user_path): + runner = CliRunner() + result = runner.invoke(cli, ["-m", "gpt-5.5", "--options"], catch_exceptions=False) + expected = runner.invoke( + cli, ["models", "-m", "gpt-5.5", "--options"], catch_exceptions=False + ) + assert result.exit_code == 0 + assert expected.exit_code == 0 + assert result.output == expected.output + assert "OpenAI Chat: gpt-5.5" in result.output + assert " Options:" in result.output + assert " reasoning_effort: str" in result.output + assert not (user_path / "logs.db").exists() + + def test_llm_models_async(user_path): runner = CliRunner() result = runner.invoke(cli, ["models", "--async"], catch_exceptions=False) From a05e14c5c08613ac7e61f072296a68f8af901426 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 May 2026 08:44:07 -0700 Subject: [PATCH 086/258] Fixed outdated test, refs #1435 --- tests/test_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_llm.py b/tests/test_llm.py index bd98d070b..fc5c1d5d5 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -529,7 +529,7 @@ def test_prompt_options_shows_selected_model_options(user_path): assert result.exit_code == 0 assert expected.exit_code == 0 assert result.output == expected.output - assert "OpenAI Chat: gpt-5.5" in result.output + assert "OpenAI Responses: gpt-5.5" in result.output assert " Options:" in result.output assert " reasoning_effort: str" in result.output assert not (user_path / "logs.db").exists() From 5a2e0a4c541f69c6dd0095058975acf2ec4dd89a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 May 2026 09:24:51 -0700 Subject: [PATCH 087/258] --hide-reasoning and hide_reasoning=True parameters (#1442) * Rename --no-reasoning flag to --hide-reasoning * hide_reasoning= Prompt parameter, plus docs * OpenAI plugin now obeys prompt.hide_reasoning --- docs/changelog.md | 6 ++- docs/help.md | 4 +- docs/plugins/advanced-model-plugins.md | 19 +++++++ docs/python-api.md | 18 +++++++ llm/cli.py | 23 ++++---- llm/default_plugins/openai_models.py | 7 ++- llm/models.py | 21 ++++++++ tests/test_cli_streaming.py | 9 ++-- tests/test_openai_responses.py | 73 ++++++++++++++++++++++++++ 9 files changed, 160 insertions(+), 20 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index b82490464..2b85f568a 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,9 @@ # Changelog +## unreleased + +- New `hide_reasoning=True` keyword argument on `model.prompt()`, `conversation.prompt()`, `model.chain()`, `conversation.chain()`, and their async counterparts, exposed to model plugins as `prompt.hide_reasoning`. Model plugins can use this to decide if they should request visible reasoning summaries from their providers. + (v0_32_a1)= ## 0.32a1 (2026-04-29) @@ -32,7 +36,7 @@ Plugin authors should read the expanded {ref}`Advanced model plugins ... key/value options for the model -d, --database FILE Path to log database --no-stream Do not stream output - -R, --no-reasoning Don't display reasoning output + -R, --hide-reasoning Hide reasoning output --key TEXT API key to use -T, --tool TEXT Name of a tool to make available to the model --functions TEXT Python code block or file path defining diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index 6b562af8d..b010491c9 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -325,6 +325,25 @@ yield StreamEvent(type="reasoning", chunk=text_chunk) Reasoning events that appear before/after text events become distinct `ReasoningPart` and `TextPart` entries in `response.messages` automatically. If your provider emits two thinking blocks separated by a tool call, you'll get two `ReasoningPart`s. +Plugins should respect `prompt.hide_reasoning`. This is set when the caller passes `hide_reasoning=True` to `model.prompt()`, `conversation.prompt()`, `model.chain()`, `conversation.chain()`, or their async counterparts. It is also set by the CLI `-R/--hide-reasoning` option. + +`prompt.hide_reasoning` means "hide visible reasoning output", not "disable model reasoning". If your provider requires an explicit request for visible reasoning summaries, do not request those summaries when `prompt.hide_reasoning` is true: + +```python +kwargs = {} +if not prompt.hide_reasoning: + kwargs["reasoning"] = {"summary": "auto"} +``` + +If your provider emits reasoning blocks regardless of request parameters, keep yielding those reasoning events as usual: + +```python +if chunk.type == "thinking": + yield StreamEvent(type="reasoning", chunk=chunk.text) +``` + +LLM's display layers use `prompt.hide_reasoning` to avoid showing those events to the user, while still allowing the framework to persist `ReasoningPart` objects and provider metadata for logs, serialization, and future turns. + ### Tool calls Each tool call emits two event types sharing a `tool_call_id`: diff --git a/docs/python-api.md b/docs/python-api.md index bbf0910b8..d28193c22 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -591,6 +591,24 @@ Event types are `"text"`, `"reasoning"`, `"tool_call_name"`, `"tool_call_args"`, Iterating against the response object itself (`for chunk in response`) yields only text strings — reasoning and tool-call events are filtered out. +#### Hiding reasoning output + +Some model plugins can return visible reasoning text, exposed as `"reasoning"` events from `response.stream_events()` and assembled as `ReasoningPart` objects in `response.messages()`. + +Pass `hide_reasoning=True` to ask LLM and supported model plugins not to expose that visible reasoning output: + +```python +response = model.prompt( + "Explain quantum computing briefly.", + hide_reasoning=True, +) +print(response.text()) +``` + +This is the Python API equivalent of the CLI `-R/--hide-reasoning` option. It is available on `model.prompt()`, `conversation.prompt()`, `model.chain()`, `conversation.chain()`, and their async counterparts. + +Note that this only requests that the underlying model does not return visible tokens. This request may not be supported by your provider, in which case this hint will not prevent visible reasoning tokens from being returned in the stream. + #### Inspecting the finished response `response.messages()` returns the assembled list of `Message` objects produced by the model, excluding the messages from the original prompt. Calling it forces execution if the response hasn't been drained yet, so you don't need a separate `response.text()` first: diff --git a/llm/cli.py b/llm/cli.py index f33a7cc76..263207252 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -489,9 +489,7 @@ def cli(): @click.option("--no-stream", is_flag=True, help="Do not stream output") @click.option("-n", "--no-log", is_flag=True, help="Don't log to database") @click.option("--log", is_flag=True, help="Log prompt and response to the database") -@click.option( - "-R", "--no-reasoning", is_flag=True, help="Don't display reasoning output" -) +@click.option("-R", "--hide-reasoning", is_flag=True, help="Hide reasoning output") @click.option( "_continue", "-c", @@ -542,7 +540,7 @@ def prompt( no_stream, no_log, log, - no_reasoning, + hide_reasoning, _continue, conversation_id, key, @@ -895,6 +893,9 @@ def read_prompt(): # Merge in options for the .prompt() methods kwargs.update(validated_options) + if hide_reasoning: + kwargs["hide_reasoning"] = True + try: if async_: @@ -911,7 +912,7 @@ async def inner(): ) await display_async_stream_events( response.astream_events(), - show_reasoning=not no_reasoning, + show_reasoning=not hide_reasoning, ) print("") else: @@ -946,7 +947,7 @@ async def inner(): if should_stream: display_stream_events( response.stream_events(), - show_reasoning=not no_reasoning, + show_reasoning=not hide_reasoning, ) print("") else: @@ -1044,9 +1045,7 @@ async def inner(): help="Path to log database", ) @click.option("--no-stream", is_flag=True, help="Do not stream output") -@click.option( - "-R", "--no-reasoning", is_flag=True, help="Don't display reasoning output" -) +@click.option("-R", "--hide-reasoning", is_flag=True, help="Hide reasoning output") @click.option("--key", help="API key to use") @click.option( "tools", @@ -1095,7 +1094,7 @@ def chat( param, options, no_stream, - no_reasoning, + hide_reasoning, key, database, tools, @@ -1197,6 +1196,8 @@ def chat( if key and isinstance(model, KeyModel): kwargs["key"] = key + if hide_reasoning: + kwargs["hide_reasoning"] = True try: fragments_and_attachments = resolve_fragments( @@ -1302,7 +1303,7 @@ def chat( argument_system_fragments = [] display_stream_events( response.stream_events(), - show_reasoning=not no_reasoning, + show_reasoning=not hide_reasoning, ) response.log_to_db(db) print("") diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 09367c0de..724497271 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1349,10 +1349,13 @@ def _build_responses_kwargs(self, prompt, stream): if seed is not None: kwargs["seed"] = seed if self._reasoning: - reasoning = {"summary": "auto"} + reasoning = {} + if not getattr(prompt, "hide_reasoning", False): + reasoning["summary"] = "auto" if reasoning_effort: reasoning["effort"] = reasoning_effort - kwargs["reasoning"] = reasoning + if reasoning: + kwargs["reasoning"] = reasoning text: Dict[str, Any] = {} if verbosity: diff --git a/llm/models.py b/llm/models.py index b929e7dc9..aefdbe05d 100644 --- a/llm/models.py +++ b/llm/models.py @@ -357,6 +357,7 @@ class Prompt: tools: List[Tool] tool_results: List[ToolResult] options: "Options" + hide_reasoning: bool def __init__( self, @@ -373,6 +374,7 @@ def __init__( tools=None, tool_results=None, messages=None, + hide_reasoning=False, ): self._prompt = prompt self.model = model @@ -387,6 +389,7 @@ def __init__( self.tools = _wrap_tools(tools or []) self.tool_results = tool_results or [] self.options = options or {} + self.hide_reasoning = hide_reasoning # Explicit messages= list, if the caller supplied one. Copied so # later mutation by the caller doesn't alter the Prompt. self._explicit_messages = list(messages) if messages is not None else None @@ -596,6 +599,7 @@ def prompt( stream: bool = True, key: Optional[str] = None, options: Optional[dict] = None, + hide_reasoning: bool = False, **kwargs, ) -> "Response": merged = _merge_options(options, kwargs) @@ -620,6 +624,7 @@ def prompt( system_fragments=system_fragments, messages=chain, options=self.model.Options(**merged), + hide_reasoning=hide_reasoning, ), self.model, stream, @@ -645,6 +650,7 @@ def chain( after_call: Optional[AfterCallSync] = None, key: Optional[str] = None, options: Optional[dict] = None, + hide_reasoning: bool = False, ) -> "ChainResponse": self.model._validate_attachments(attachments) # Parity with Conversation.prompt: pre-bake the full chain so @@ -670,6 +676,7 @@ def chain( messages=chain_messages, model=self.model, options=self.model.Options(**(options or {})), + hide_reasoning=hide_reasoning, ), model=self.model, stream=stream, @@ -719,6 +726,7 @@ def chain( after_call: Optional[AfterCallAsync] = None, key: Optional[str] = None, options: Optional[dict] = None, + hide_reasoning: bool = False, ) -> "AsyncChainResponse": self.model._validate_attachments(attachments) chain_messages = self._build_full_chain( @@ -740,6 +748,7 @@ def chain( messages=chain_messages, model=self.model, options=self.model.Options(**(options or {})), + hide_reasoning=hide_reasoning, ), model=self.model, stream=stream, @@ -765,6 +774,7 @@ def prompt( stream: bool = True, key: Optional[str] = None, options: Optional[dict] = None, + hide_reasoning: bool = False, **kwargs, ) -> "AsyncResponse": merged = _merge_options(options, kwargs) @@ -787,6 +797,7 @@ def prompt( system_fragments=system_fragments, messages=chain, options=self.model.Options(**merged), + hide_reasoning=hide_reasoning, ), self.model, stream, @@ -2570,6 +2581,7 @@ def responses(self) -> Iterator[Response]: system_fragments=self.prompt.system_fragments, options=self.prompt.options, attachments=attachments, + hide_reasoning=current_response.prompt.hide_reasoning, ), self.model, stream=self.stream, @@ -2641,6 +2653,7 @@ async def responses(self) -> AsyncIterator[AsyncResponse]: system_fragments=self.prompt.system_fragments, options=self.prompt.options, attachments=attachments, + hide_reasoning=current_response.prompt.hide_reasoning, ) current_response = AsyncResponse( prompt, @@ -2777,6 +2790,7 @@ def prompt( tools: Optional[List[ToolDef]] = None, tool_results: Optional[List[ToolResult]] = None, options: Optional[dict] = None, + hide_reasoning: bool = False, **kwargs, ) -> Response: key_value = kwargs.pop("key", None) @@ -2795,6 +2809,7 @@ def prompt( messages=messages, model=self, options=self.Options(**merged), + hide_reasoning=hide_reasoning, ), self, stream, @@ -2818,6 +2833,7 @@ def chain( after_call: Optional[AfterCallSync] = None, key: Optional[str] = None, options: Optional[dict] = None, + hide_reasoning: bool = False, ) -> ChainResponse: return self.conversation().chain( prompt=prompt, @@ -2834,6 +2850,7 @@ def chain( after_call=after_call, key=key, options=options, + hide_reasoning=hide_reasoning, ) @@ -2892,6 +2909,7 @@ def prompt( messages: Optional[List[Any]] = None, stream: bool = True, options: Optional[dict] = None, + hide_reasoning: bool = False, **kwargs, ) -> AsyncResponse: key_value = kwargs.pop("key", None) @@ -2910,6 +2928,7 @@ def prompt( messages=messages, model=self, options=self.Options(**merged), + hide_reasoning=hide_reasoning, ), self, stream, @@ -2933,6 +2952,7 @@ def chain( after_call: Optional[AfterCallAsync] = None, key: Optional[str] = None, options: Optional[dict] = None, + hide_reasoning: bool = False, ) -> AsyncChainResponse: return self.conversation().chain( prompt=prompt, @@ -2949,6 +2969,7 @@ def chain( after_call=after_call, key=key, options=options, + hide_reasoning=hide_reasoning, ) diff --git a/tests/test_cli_streaming.py b/tests/test_cli_streaming.py index c8efee2fe..e83586881 100644 --- a/tests/test_cli_streaming.py +++ b/tests/test_cli_streaming.py @@ -1,5 +1,5 @@ """Tests for CLI streaming display: reasoning → stderr (dim), -text → stdout, -R / --no-reasoning flag. +text → stdout, -R / --hide-reasoning flag. """ import click @@ -65,7 +65,7 @@ def test_reasoning_rendered_in_dim_style(mock_model): assert dim_start in result.stderr -def test_no_reasoning_flag_suppresses_reasoning(mock_model): +def test_hide_reasoning_flag_suppresses_reasoning(mock_model): mock_model.enqueue( [ llm.parts.StreamEvent( @@ -77,16 +77,17 @@ def test_no_reasoning_flag_suppresses_reasoning(mock_model): runner = CliRunner(mix_stderr=False) result = runner.invoke( cli, - ["-m", "mock", "hi", "--no-log", "--no-reasoning"], + ["-m", "mock", "hi", "--no-log", "--hide-reasoning"], catch_exceptions=False, ) assert result.exit_code == 0 assert "hidden thinking" not in result.stderr assert "hidden thinking" not in result.stdout assert "answer" in result.stdout + assert mock_model.history[0][0].hide_reasoning is True -def test_no_reasoning_short_flag_R(mock_model): +def test_hide_reasoning_short_flag_R(mock_model): mock_model.enqueue( [ llm.parts.StreamEvent(type="reasoning", chunk="hidden", part_index=0), diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index 1527e0875..50b981577 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -140,6 +140,47 @@ def test_default_routes_to_responses_endpoint(httpx_mock): assert request_body["reasoning"] == {"summary": "auto"} +def test_hide_reasoning_omits_reasoning_summary_from_responses_request(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_test_1", + "object": "response", + "created_at": 1, + "model": "gpt-5.5", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "hidden", + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + }, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + model = llm.get_model("gpt-5.5") + response = model.prompt("hello", stream=False, key="test", hide_reasoning=True) + assert response.text() == "hidden" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["include"] == ["reasoning.encrypted_content"] + assert "reasoning" not in request_body + + def test_non_reasoning_responses_model_omits_encrypted_reasoning_include(httpx_mock): from llm.default_plugins.openai_models import Responses @@ -349,6 +390,38 @@ class FakePrompt: assert kwargs["reasoning"] == {"summary": "auto"} +def test_responses_kwargs_omits_reasoning_summary_when_hide_reasoning(): + model = llm.get_model("gpt-5.5") + options = model.Options(reasoning_effort="low") + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + p.hide_reasoning = True + kwargs = model._build_responses_kwargs(p, stream=False) + assert kwargs["reasoning"] == {"effort": "low"} + + +def test_responses_kwargs_omits_empty_reasoning_when_hide_reasoning(): + model = llm.get_model("gpt-5.5") + options = model.Options() + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + p.hide_reasoning = True + kwargs = model._build_responses_kwargs(p, stream=False) + assert "reasoning" not in kwargs + + def test_responses_streams_reasoning_summary_text(httpx_mock): httpx_mock.add_response( method="POST", From 8aba606447dc2fb0e7a31aa942a6477c7ac4d088 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 May 2026 10:42:55 -0700 Subject: [PATCH 088/258] Release 0.32a2 Refs #1432, #1433, #1435, #1441, #1442 --- docs/changelog.md | 28 ++++++++++++++++++++++++-- docs/fragments.md | 2 +- docs/plugins/advanced-model-plugins.md | 1 + pyproject.toml | 2 +- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 2b85f568a..cb2d5a78b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,8 +1,32 @@ # Changelog -## unreleased +(v0_32_a2)= +## 0.32a2 (2026-05-12) -- New `hide_reasoning=True` keyword argument on `model.prompt()`, `conversation.prompt()`, `model.chain()`, `conversation.chain()`, and their async counterparts, exposed to model plugins as `prompt.hide_reasoning`. Model plugins can use this to decide if they should request visible reasoning summaries from their providers. +### Support for the OpenAI Responses API + +Most reasoning-capable OpenAI models now use the [`/v1/responses`](https://platform.openai.com/docs/api-reference/responses) endpoint instead of `/v1/chat/completions`. This enables interleaved reasoning across tool calls for GPT-5 class models. [#1435](https://github.com/simonw/llm/pull/1435) + +- New `Responses` and `AsyncResponses` model classes driving the OpenAI Responses API. The existing `Chat` and `AsyncChat` classes are unchanged so other plugins that import them keep working. +- The following models now use the Responses API by default: `o1`, `o3-mini`, `o3`, `o4-mini`, `gpt-5`, `gpt-5-mini`, `gpt-5-nano`, `gpt-5.1`, `gpt-5.2`, `gpt-5.4`, `gpt-5.4-mini`, `gpt-5.4-nano`, `gpt-5.5` (and their pinned date variants). +- Use `-o chat_completions 1` to fall back to the older `/v1/chat/completions` code path for any of these models. +- Encrypted reasoning items are captured as `provider_metadata` on `ReasoningPart` objects and round-tripped back to OpenAI on subsequent turns. +- Reasoning summaries are now requested with `"summary": "auto"` so visible reasoning text is streamed back where the model produces it, unless `--hide-reasoning` or `hide_reasoning=` is set. +- This means OpenAI prompts run using `llm prompt` that return reasoning tokens will display those on standard error. + +### CLI + +- New `llm -m model --options` flag to list the options supported by a given model. [#1441](https://github.com/simonw/llm/pull/1441) +- The `-R/--no-reasoning` option has been renamed to `-R/--hide-reasoning`. + +### Python API + +- New `hide_reasoning=True` keyword argument on `model.prompt()`, `conversation.prompt()`, `model.chain()`, `conversation.chain()`, and their async counterparts, exposed to model plugins as `prompt.hide_reasoning`. Model plugins can {ref}`use this to decide ` if they should request visible reasoning summaries from their providers. [#1442](https://github.com/simonw/llm/pull/1442) +- New `options=` dict keyword argument on `Model.prompt()`, `Conversation.prompt()`, `Response.reply()`, and their async equivalents, matching the pattern already used by `.chain()`. The previous `**kwargs` form continues to work for backwards compatibility but is no longer documented, and will be removed in the future. [#1432](https://github.com/simonw/llm/pull/1432) + +### Bug fixes + +- `add_tool_call()` calls that were not also recorded as stream events are now correctly emitted as `ToolCallPart` objects when assembling response parts, so they survive serialization via `response.to_dict()`. [#1433](https://github.com/simonw/llm/issues/1433) (v0_32_a1)= ## 0.32a1 (2026-04-29) diff --git a/docs/fragments.md b/docs/fragments.md index 935bbab32..318a893f5 100644 --- a/docs/fragments.md +++ b/docs/fragments.md @@ -30,7 +30,7 @@ from importlib.metadata import version llm_version = version("llm") cog.out(f'The URL will be fetched with the user-agent `llm/{llm_version} (https://llm.datasette.io/)`.') ]]]--> -The URL will be fetched with the user-agent `llm/0.32a1 (https://llm.datasette.io/)`. +The URL will be fetched with the user-agent `llm/0.32a2 (https://llm.datasette.io/)`. The `-f` option can be used multiple times to combine together multiple fragments. diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index b010491c9..1cfa54f93 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -315,6 +315,7 @@ In rare cases you'll want to override the default grouping: You can mix explicit indices with `None` in the same stream — the framework reserves your explicit values and decides the rest. +(advanced-model-plugins-reasoning-tokens)= ### Reasoning tokens For streamed reasoning text: diff --git a/pyproject.toml b/pyproject.toml index 948302833..9725cdfa7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm" -version = "0.32a1" +version = "0.32a2" description = "CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine." readme = { file = "README.md", content-type = "text/markdown" } authors = [ From be27b91aa83142ee080c467976dc47fa4b5fb7ec Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 16 May 2026 18:38:18 -0700 Subject: [PATCH 089/258] Fixed broken link --- docs/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index cb2d5a78b..40aefd2fe 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -5,7 +5,7 @@ ### Support for the OpenAI Responses API -Most reasoning-capable OpenAI models now use the [`/v1/responses`](https://platform.openai.com/docs/api-reference/responses) endpoint instead of `/v1/chat/completions`. This enables interleaved reasoning across tool calls for GPT-5 class models. [#1435](https://github.com/simonw/llm/pull/1435) +Most reasoning-capable OpenAI models now use the [`/v1/responses`](https://developers.openai.com/api/reference/responses/overview) endpoint instead of `/v1/chat/completions`. This enables interleaved reasoning across tool calls for GPT-5 class models. [#1435](https://github.com/simonw/llm/pull/1435) - New `Responses` and `AsyncResponses` model classes driving the OpenAI Responses API. The existing `Chat` and `AsyncChat` classes are unchanged so other plugins that import them keep working. - The following models now use the Responses API by default: `o1`, `o3-mini`, `o3`, `o4-mini`, `gpt-5`, `gpt-5-mini`, `gpt-5-nano`, `gpt-5.1`, `gpt-5.2`, `gpt-5.4`, `gpt-5.4-mini`, `gpt-5.4-nano`, `gpt-5.5` (and their pinned date variants). From b865ede0f17f12bc22a1159ab9e3f8110401d2cb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 Jun 2026 13:51:39 -0700 Subject: [PATCH 090/258] Tool implementations can receive the ToolCall via llm_tool_call param (#1480) * Tool implementations can receive the ToolCall via llm_tool_call parameter Tool functions (sync or async, including Toolbox methods) that declare a parameter named llm_tool_call are now passed the llm.ToolCall object for the current invocation. The parameter is reserved: it is excluded from the input schema exposed to the model and is only injected when declared explicitly - a **kwargs catch-all does not receive it. This lets tool implementations key external state against the unique tool_call_id, e.g. for human-in-the-loop approval flows that need to resume a specific tool call after the answer arrives. Co-Authored-By: Claude Fable 5 * Ran Black --------- Co-authored-by: Claude Fable 5 --- docs/python-api.md | 20 +++++++ docs/tools.md | 2 + llm/models.py | 37 ++++++++++-- tests/test_tools.py | 138 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 192 insertions(+), 5 deletions(-) diff --git a/docs/python-api.md b/docs/python-api.md index d28193c22..8d2424b24 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -211,6 +211,26 @@ response = model.chain( print(response.text()) ``` +(python-api-tools-llm-tool-call)= + +#### Accessing the tool call from inside a tool + +Tool implementations sometimes need to know about the `llm.ToolCall` that triggered them - most often the `tool_call_id`, which can be used to key external state against that specific invocation. + +If your tool function accepts a parameter named `llm_tool_call` it will be passed the `llm.ToolCall` object for the current call: + +```python +import llm + +def lookup(name: str, llm_tool_call: llm.ToolCall) -> str: + "Look up a name." + return do_lookup(name, request_id=llm_tool_call.tool_call_id) +``` + +The `llm_tool_call` parameter name is reserved: it is excluded from the input schema that is exposed to the model and is populated automatically when the tool executes. The type annotation is optional. + +This works for both sync and async tool functions, and for methods on `llm.Toolbox` subclasses. The parameter must be declared explicitly - a `**kwargs` catch-all will not receive `llm_tool_call`. + (python-api-tools-attachments)= #### Tools can return attachments diff --git a/docs/tools.md b/docs/tools.md index a67600745..945ae8dbd 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -97,4 +97,6 @@ Consult the {ref}`register_tools() plugin hook ` do If your plugin needs access to API secrets I recommend storing those using `llm keys set api-name` and then reading them using the {ref}`plugin-utilities-get-key` utility function. This avoids secrets being logged to the database as part of tool calls. +If your tool implementation needs to know which tool call invoked it - for example to key state against the unique `tool_call_id` - add a parameter named `llm_tool_call` to your function. It will be passed the `llm.ToolCall` object for the current invocation, and is hidden from the schema the model sees. See {ref}`python-api-tools-llm-tool-call` for details. + diff --git a/llm/models.py b/llm/models.py index aefdbe05d..ab333da80 100644 --- a/llm/models.py +++ b/llm/models.py @@ -188,7 +188,9 @@ def _get_arguments_input_schema(function, name): type_hints = get_type_hints(function) fields = {} for param_name, param in signature.parameters.items(): - if param_name == "self": + if param_name in ("self", "llm_tool_call"): + # llm_tool_call is reserved: populated with the ToolCall object + # at execution time, never exposed to the model. continue # Determine the type annotation (default to string if missing) annotated_type = type_hints.get(param_name, str) @@ -202,6 +204,26 @@ def _get_arguments_input_schema(function, name): return create_model(f"{name}InputSchema", **fields) +def _accepts_llm_tool_call(implementation) -> bool: + try: + signature = inspect.signature(implementation) + except (TypeError, ValueError): + return False + return "llm_tool_call" in signature.parameters + + +def _implementation_arguments(tool: "Tool", tool_call: "ToolCall") -> dict: + """Arguments to invoke a tool implementation with. + + Implementations with an explicit ``llm_tool_call`` parameter receive + the ToolCall object itself - a ``**kwargs`` catch-all does not count. + """ + arguments = dict(tool_call.arguments) + if _accepts_llm_tool_call(tool.implementation): + arguments["llm_tool_call"] = tool_call + return arguments + + class Toolbox: name: Optional[str] = None instance_id: Optional[int] = None @@ -1814,10 +1836,13 @@ def execute_tool_calls( exception = None try: + implementation_arguments = _implementation_arguments(tool, tool_call) if inspect.iscoroutinefunction(tool.implementation): - result = asyncio.run(tool.implementation(**tool_call.arguments)) + result = asyncio.run( + tool.implementation(**implementation_arguments) + ) else: - result = tool.implementation(**tool_call.arguments) + result = tool.implementation(**implementation_arguments) if isinstance(result, ToolOutput): attachments = result.attachments @@ -2127,7 +2152,9 @@ async def run_async(tc=tc, tool=tool, idx=idx): attachments = [] try: - result = await tool.implementation(**tc.arguments) + result = await tool.implementation( + **_implementation_arguments(tool, tc) + ) if isinstance(result, ToolOutput): attachments.extend(result.attachments) result = result.output @@ -2188,7 +2215,7 @@ async def run_async(tc=tc, tool=tool, idx=idx): exception = KeyError(tc.name) else: try: - res = tool.implementation(**tc.arguments) + res = tool.implementation(**_implementation_arguments(tool, tc)) if inspect.isawaitable(res): res = await res if isinstance(res, ToolOutput): diff --git a/tests/test_tools.py b/tests/test_tools.py index b849779cc..a9a321280 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -592,3 +592,141 @@ async def before(tool, tool_call): assert results[1].name == "t2" assert results[1].output == "ran2" assert results[1].exception is None + + +def test_tool_function_receives_llm_tool_call(): + captured = {} + + def lookup(name: str, llm_tool_call) -> str: + "Look up a name" + captured["tool_call"] = llm_tool_call + return "result for " + name + + model = llm.get_model("echo") + chain_response = model.chain( + json.dumps( + {"tool_calls": [{"name": "lookup", "arguments": {"name": "simon"}}]} + ), + tools=[lookup], + ) + chain_response.text() + + tool_call = captured["tool_call"] + assert isinstance(tool_call, llm.ToolCall) + assert tool_call.name == "lookup" + assert tool_call.arguments == {"name": "simon"} + second = chain_response._responses[1] + assert second.prompt.tool_results[0].output == "result for simon" + + +def test_async_tool_function_receives_llm_tool_call_with_sync_model(): + captured = {} + + async def lookup(name: str, llm_tool_call: llm.ToolCall) -> str: + "Look up a name" + captured["tool_call"] = llm_tool_call + return "result for " + name + + model = llm.get_model("echo") + chain_response = model.chain( + json.dumps( + {"tool_calls": [{"name": "lookup", "arguments": {"name": "simon"}}]} + ), + tools=[lookup], + ) + chain_response.text() + + tool_call = captured["tool_call"] + assert isinstance(tool_call, llm.ToolCall) + assert tool_call.name == "lookup" + assert tool_call.arguments == {"name": "simon"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_tool", (False, True)) +async def test_tool_function_receives_llm_tool_call_async_model(async_tool): + captured = {} + + def lookup(name: str, llm_tool_call) -> str: + "Look up a name" + captured["tool_call"] = llm_tool_call + return "result for " + name + + async def async_lookup(name: str, llm_tool_call) -> str: + "Look up a name" + captured["tool_call"] = llm_tool_call + return "result for " + name + + fn = async_lookup if async_tool else lookup + model = llm.get_async_model("echo") + chain_response = model.chain( + json.dumps( + {"tool_calls": [{"name": fn.__name__, "arguments": {"name": "simon"}}]} + ), + tools=[fn], + ) + output = await chain_response.text() + assert '"output": "result for simon"' in output + + tool_call = captured["tool_call"] + assert isinstance(tool_call, llm.ToolCall) + assert tool_call.name == fn.__name__ + assert tool_call.arguments == {"name": "simon"} + + +def test_llm_tool_call_excluded_from_input_schema(): + def lookup(name: str, llm_tool_call) -> str: + "Look up a name" + return name + + tool = llm.Tool.function(lookup) + assert "llm_tool_call" not in tool.input_schema.get("properties", {}) + assert "llm_tool_call" not in tool.input_schema.get("required", []) + assert "name" in tool.input_schema["properties"] + + +def test_kwargs_only_function_does_not_receive_llm_tool_call(): + # A tool that accepts **kwargs but does not name llm_tool_call + # explicitly should NOT have it injected. + captured = {} + + async def impl(**kwargs): + captured.update(kwargs) + return "ok" + + tool = llm.Tool( + name="t", + description="A tool", + input_schema={"type": "object", "properties": {"name": {"type": "string"}}}, + implementation=impl, + ) + model = llm.get_model("echo") + chain_response = model.chain( + json.dumps({"tool_calls": [{"name": "t", "arguments": {"name": "x"}}]}), + tools=[tool], + ) + chain_response.text() + assert captured == {"name": "x"} + + +def test_toolbox_method_receives_llm_tool_call(): + captured = {} + + class Tools(llm.Toolbox): + def lookup(self, name: str, llm_tool_call) -> str: + captured["tool_call"] = llm_tool_call + return "hi " + name + + model = llm.get_model("echo") + chain_response = model.chain( + json.dumps( + {"tool_calls": [{"name": "Tools_lookup", "arguments": {"name": "simon"}}]} + ), + tools=[Tools()], + ) + output = chain_response.text() + assert '"output": "hi simon"' in output + + tool_call = captured["tool_call"] + assert isinstance(tool_call, llm.ToolCall) + assert tool_call.arguments == {"name": "simon"} From 73bb0221b2df44ba3988d775ff82221e44c64206 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 Jun 2026 14:04:26 -0700 Subject: [PATCH 091/258] Guaranteed tool call IDs (#1481) * Guarantee every tool call has a unique tool_call_id add_tool_call() now synthesizes a unique tc_-prefixed id (monotonic ULID) whenever the provider did not supply one. Previously consumers correlating tool calls with results - or keying external state on a specific invocation - had to invent fallback matching schemes for id-less providers, and test models like llm-echo exercised different code paths than production providers. Provider-supplied ids are preserved untouched, and responses rehydrated from the logs database keep their stored ids (synthesis only happens at add_tool_call time). Existing tests that asserted tool_call_id None now normalize or mask the synthesized ids. Co-Authored-By: Claude Fable 5 --- docs/plugins/advanced-model-plugins.md | 2 +- docs/python-api.md | 3 +- llm/models.py | 10 +++ tests/test_chat.py | 6 +- tests/test_llm_logs.py | 5 +- tests/test_plugins.py | 27 +++++--- tests/test_tools.py | 94 +++++++++++++++++++++++++- 7 files changed, 129 insertions(+), 18 deletions(-) diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index 1cfa54f93..3a1e2b939 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -138,7 +138,7 @@ Adding {ref}`tools support ` involves several steps: 1. Add `supports_tools = True` to your model class. 2. If `prompt.tools` is populated, turn that list of `llm.Tool` objects into the correct format for your model. -3. Look out for requests to call tools in the responses from your model. Call `response.add_tool_call(llm.ToolCall(...))` for each of those. This should work for streaming and non-streaming and async and non-async cases. +3. Look out for requests to call tools in the responses from your model. Call `response.add_tool_call(llm.ToolCall(...))` for each of those. This should work for streaming and non-streaming and async and non-async cases. Pass the provider's tool call ID as `tool_call_id=` if there is one; if you omit it LLM synthesizes a unique `tc_`-prefixed id, since consumers rely on every tool call having one. 4. If your prompt has a `prompt.tool_results` list, pass the information from those `llm.ToolResult` objects to your model. 5. Include `prompt.tools` and `prompt.tool_results` and tool calls from `response.tool_calls_or_raise()` in the conversation history constructed by your plugin. 6. Make sure your code is OK with prompts that do not have `prompt.prompt` set to a value, since they may be carrying exclusively the results of a tool call. diff --git a/docs/python-api.md b/docs/python-api.md index 8d2424b24..7186701ae 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -116,6 +116,7 @@ response = model.prompt("Convert panda to upper", tools=[upper]) tool_calls = response.tool_calls() # [ToolCall(name='upper', arguments={'text': 'panda'}, tool_call_id='...')] ``` +Every tool call is guaranteed to have a unique `tool_call_id`. Most providers supply their own; for providers that do not, LLM synthesizes one of the form `tc_01...`, so you can always use the id to correlate a tool call with its result or to key external state against a specific invocation. You can call `response.execute_tool_calls()` to execute those calls and get back the results: ```python tool_results = response.execute_tool_calls() @@ -215,7 +216,7 @@ print(response.text()) #### Accessing the tool call from inside a tool -Tool implementations sometimes need to know about the `llm.ToolCall` that triggered them - most often the `tool_call_id`, which can be used to key external state against that specific invocation. +Tool implementations sometimes need to know about the `llm.ToolCall` that triggered them - most often the `tool_call_id` (always populated, see above), which can be used to key external state against that specific invocation. If your tool function accepts a parameter named `llm_tool_call` it will be passed the `llm.ToolCall` object for the current call: diff --git a/llm/models.py b/llm/models.py index ab333da80..04658ff1e 100644 --- a/llm/models.py +++ b/llm/models.py @@ -1,6 +1,7 @@ import asyncio import base64 from condense_json import condense_json +import dataclasses from dataclasses import dataclass, field import datetime from .errors import NeedsKeyException @@ -1223,6 +1224,15 @@ def _build_parts(self) -> List[Any]: return parts def add_tool_call(self, tool_call: ToolCall): + if tool_call.tool_call_id is None: + # Guarantee every locally-executable tool call has a unique id. + # Some providers never supply one, which otherwise forces every + # consumer correlating calls with results (or keying external + # state on a call) to invent fallback matching schemes. + tool_call = dataclasses.replace( + tool_call, + tool_call_id="tc_{}".format(str(monotonic_ulid()).lower()), + ) self._tool_calls.append(tool_call) def set_usage( diff --git a/tests/test_chat.py b/tests/test_chat.py index 5d6089cb0..dbd77ab4e 100644 --- a/tests/test_chat.py +++ b/tests/test_chat.py @@ -1,4 +1,5 @@ from click.testing import CliRunner +import re from unittest.mock import ANY import json import llm.cli @@ -338,7 +339,8 @@ def upper(text: str) -> str: catch_exceptions=False, ) assert result.exit_code == 0 - assert result.output == ( + normalized_output = re.sub(r"tc_[0-9a-z]{26}", "tc_TCID", result.output) + assert normalized_output == ( "Chatting with echo\n" "Type 'exit' or 'quit' to exit\n" "Type '!multi' to enter multiple lines, then '!end' to finish\n" @@ -368,7 +370,7 @@ def upper(text: str) -> str: " {\n" ' "name": "upper",\n' ' "output": "HELLO",\n' - ' "tool_call_id": null\n' + ' "tool_call_id": "tc_TCID"\n' " }\n" " ]\n" "}\n" diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index f2f6167c6..78b160b08 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -950,15 +950,16 @@ def demo(): ) assert result1.exit_code == 0 result2 = runner.invoke(cli, ["logs", "-c"]) + normalized_output = re.sub(r"tc_[0-9a-z]{26}", "tc_TCID", result2.output) assert ( "### Tool results\n" "\n" - "- **demo**: `None`
\n" + "- **demo**: `tc_TCID`
\n" " one\n" " two\n" " three\n" "\n" - ) in result2.output + ) in normalized_output # Log one that did NOT use tools, check that `llm logs --tools` ignores it assert runner.invoke(cli, ["-m", "echo", "badger"]).exit_code == 0 assert "badger" in runner.invoke(cli, ["logs"]).output diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 6777fd585..6197ef5e2 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -6,6 +6,8 @@ from llm.tools import llm_version, llm_time from llm import cli, hookimpl, plugins, get_template_loaders, get_fragment_loaders import pathlib +import re +from unittest.mock import ANY import pytest import textwrap @@ -450,19 +452,24 @@ def register_tools(self, register): runner.invoke(cli.cli, ["logs", "-c", "-n", "0", "--json"]).output ) results = tuple( - (log_row["prompt"], json.dumps(log_row["tool_results"])) + ( + log_row["prompt"], + re.sub( + r"tc_[0-9a-z]{26}", "tc_TCID", json.dumps(log_row["tool_results"]) + ), + ) for log_row in log_rows ) assert results == ( ('{"tool_calls": [{"name": "upper", "arguments": {"text": "one"}}]}', "[]"), ( "", - '[{"id": 2, "tool_id": 1, "name": "upper", "output": "ONE", "tool_call_id": null, "exception": null, "attachments": []}]', + '[{"id": 2, "tool_id": 1, "name": "upper", "output": "ONE", "tool_call_id": "tc_TCID", "exception": null, "attachments": []}]', ), ('{"tool_calls": [{"name": "upper", "arguments": {"text": "two"}}]}', "[]"), ( "", - '[{"id": 3, "tool_id": 1, "name": "upper", "output": "TWO", "tool_call_id": null, "exception": null, "attachments": []}]', + '[{"id": 3, "tool_id": 1, "name": "upper", "output": "TWO", "tool_call_id": "tc_TCID", "exception": null, "attachments": []}]', ), ( '{"tool_calls": [{"name": "upper", "arguments": {"text": "three"}}]}', @@ -470,7 +477,7 @@ def register_tools(self, register): ), ( "", - '[{"id": 4, "tool_id": 1, "name": "upper", "output": "THREE", "tool_call_id": null, "exception": null, "attachments": []}]', + '[{"id": 4, "tool_id": 1, "name": "upper", "output": "THREE", "tool_call_id": "tc_TCID", "exception": null, "attachments": []}]', ), ) # Test the --td option @@ -739,8 +746,8 @@ def after_call(tool, tool_call, tool_result): "[" + result3.output.split('"tool_results": [')[1].split("]")[0] + "]" ) assert tool_results == [ - {"name": "Memory_set", "output": "null", "tool_call_id": None}, - {"name": "Memory_get", "output": "two", "tool_call_id": None}, + {"name": "Memory_set", "output": "null", "tool_call_id": ANY}, + {"name": "Memory_get", "output": "two", "tool_call_id": ANY}, ] # Test the CLI running a configured toolbox prompt @@ -767,7 +774,7 @@ def after_call(tool, tool_call, tool_result): { "name": "Filesystem_list_files", "output": json.dumps([str(other_path)]), - "tool_call_id": None, + "tool_call_id": ANY, } ] @@ -909,9 +916,9 @@ def test_toolbox_logging_async(logs_db, tmpdir): "[" + result.output.split('"tool_results": [')[1].rsplit("]", 1)[0] + "]" ) assert tool_results == [ - {"name": "Memory_set", "output": "null", "tool_call_id": None}, - {"name": "Memory_get", "output": "two", "tool_call_id": None}, - {"name": "Filesystem_list_files", "output": "[]", "tool_call_id": None}, + {"name": "Memory_set", "output": "null", "tool_call_id": ANY}, + {"name": "Memory_get", "output": "two", "tool_call_id": ANY}, + {"name": "Filesystem_list_files", "output": "[]", "tool_call_id": ANY}, ] finally: plugins.pm.unregister(name="ToolboxPlugin") diff --git a/tests/test_tools.py b/tests/test_tools.py index a9a321280..919aec071 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -1,4 +1,5 @@ import asyncio +import re from click.testing import CliRunner from importlib.metadata import version import json @@ -114,6 +115,9 @@ async def hello(): bits = output.split("\n}{\n") assert len(bits) == 2 objects = [json.loads(bits[0] + "}"), json.loads("{" + bits[1])] + tool_call_id = objects[1]["tool_results"][0]["tool_call_id"] + assert tool_call_id.startswith("tc_") + objects[1]["tool_results"][0]["tool_call_id"] = None assert objects == [ {"prompt": "", "system": "", "attachments": [], "stream": True, "previous": []}, { @@ -155,6 +159,11 @@ async def hello2(): bits = output.split("\n}{\n") assert len(bits) == 2 objects = [json.loads(bits[0] + "}"), json.loads("{" + bits[1])] + ids = [r["tool_call_id"] for r in objects[1]["tool_results"]] + assert all(i.startswith("tc_") for i in ids) + assert len(set(ids)) == 2 + for r in objects[1]["tool_results"]: + r["tool_call_id"] = None assert objects == [ {"prompt": "", "system": "", "attachments": [], "stream": True, "previous": []}, { @@ -512,11 +521,12 @@ def test_tool_errors(async_): # llm logs -c output log_text_result = runner.invoke(cli.cli, ["logs", "-c"]) assert log_text_result.exit_code == 0 + normalized_log_text = re.sub(r"tc_[0-9a-z]{26}", "tc_TCID", log_text_result.output) assert ( - "- **trigger_error**: `None`
\n" + "- **trigger_error**: `tc_TCID`
\n" " Error: Error!
\n" " **Error**: Exception: Error!\n" - ) in log_text_result.output + ) in normalized_log_text def test_chain_sync_cancel_only_first_of_two(): @@ -730,3 +740,83 @@ def lookup(self, name: str, llm_tool_call) -> str: tool_call = captured["tool_call"] assert isinstance(tool_call, llm.ToolCall) assert tool_call.arguments == {"name": "simon"} + + +def test_add_tool_call_synthesizes_missing_tool_call_id(): + model = llm.get_model("echo") + response = model.prompt("hello") + response.add_tool_call(llm.ToolCall(name="a", arguments={})) + response.add_tool_call(llm.ToolCall(name="b", arguments={}, tool_call_id="given")) + response.add_tool_call(llm.ToolCall(name="c", arguments={})) + ids = [tc.tool_call_id for tc in response._tool_calls] + assert ids[0] is not None and ids[0].startswith("tc_") + assert ids[1] == "given" + assert ids[2] is not None and ids[2].startswith("tc_") + assert ids[0] != ids[2] + + +def test_tool_call_ids_guaranteed_through_chain(): + seen_before_call = [] + captured = {} + + def first(llm_tool_call) -> str: + captured["first_id"] = llm_tool_call.tool_call_id + return "one" + + def second() -> str: + return "two" + + def before(tool, tool_call): + seen_before_call.append(tool_call.tool_call_id) + + model = llm.get_model("echo") + chain_response = model.chain( + json.dumps({"tool_calls": [{"name": "first"}, {"name": "second"}]}), + tools=[first, second], + before_call=before, + ) + chain_response.text() + + assert len(seen_before_call) == 2 + assert all(i is not None and i.startswith("tc_") for i in seen_before_call) + assert seen_before_call[0] != seen_before_call[1] + # The implementation saw the same id via llm_tool_call + assert captured["first_id"] == seen_before_call[0] + + # ToolResults and the next prompt's tool message carry the same ids + second_response = chain_response._responses[1] + result_ids = [r.tool_call_id for r in second_response.prompt.tool_results] + assert result_ids == seen_before_call + + # The assistant message parts carry the synthesized ids too, so a + # persisted-and-replayed history stays correlated + from llm.parts import ToolCallPart + + first_response = chain_response._responses[0] + part_ids = [ + p.tool_call_id + for p in first_response._messages_now()[0].parts + if isinstance(p, ToolCallPart) + ] + assert part_ids == seen_before_call + + +@pytest.mark.asyncio +async def test_tool_call_ids_guaranteed_async_model(): + seen = [] + + async def hello() -> str: + return "world" + + async def before(tool, tool_call): + seen.append(tool_call.tool_call_id) + + model = llm.get_async_model("echo") + chain_response = model.chain( + json.dumps({"tool_calls": [{"name": "hello"}]}), + tools=[hello], + before_call=before, + ) + await chain_response.text() + assert len(seen) == 1 + assert seen[0] is not None and seen[0].startswith("tc_") From 3ac0a233816bda9e67d0f11a72ce53ca683a30cb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 Jun 2026 15:07:27 -0700 Subject: [PATCH 092/258] PauseChain primitive + chain resume from pending tool calls (#1482) * PauseChain primitive + chain resume from pending tool calls Two features that together give chains a first-class suspend/resume story for human-in-the-loop tools: llm.PauseChain: raise inside a tool implementation to stop the chain cleanly. Unlike other exceptions it is not converted into an error ToolResult - it propagates to the caller with .tool_call (the paused call) and .tool_results (completed sibling results) attached, and no provider call is made with a placeholder result. Failure semantics for concurrent tool execution are now defined: async sibling tasks always run to completion before a pause or hook exception propagates (gather with return_exceptions, raised after collection), so nothing is orphaned mid-flight; sync execution stops at the paused call, leaving later calls unstarted so they can safely run on resume. Chain resume: chain(messages=history, tools=...) now detects a history ending in an assistant message with unresolved tool calls - e.g. one persisted when a previous run paused or crashed - executes those calls through the normal before_call/after_call machinery (skipping any that already have results), then sends the results to the model as a standard tool-result turn. A resumed call may pause again, enabling multi-question flows. Histories where a user or assistant message follows the calls are left alone. Also adds execute_tool_calls(tool_calls_list=) for executing an explicit list. Co-Authored-By: Claude Fable 5 --- docs/python-api.md | 50 +++++ llm/__init__.py | 2 + llm/models.py | 296 +++++++++++++++++++++++++--- tests/test_pause_resume.py | 391 +++++++++++++++++++++++++++++++++++++ 4 files changed, 710 insertions(+), 29 deletions(-) create mode 100644 tests/test_pause_resume.py diff --git a/docs/python-api.md b/docs/python-api.md index 7186701ae..a0929dbca 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -232,6 +232,56 @@ The `llm_tool_call` parameter name is reserved: it is excluded from the input sc This works for both sync and async tool functions, and for methods on `llm.Toolbox` subclasses. The parameter must be declared explicitly - a `**kwargs` catch-all will not receive `llm_tool_call`. +(python-api-tools-pause)= + +#### Pausing a chain from inside a tool + +Sometimes a tool cannot finish without outside input - human approval being the classic case. Raise `llm.PauseChain` inside a tool implementation to stop the chain cleanly: + +```python +import llm + +def delete_files(path: str) -> str: + if not approval_already_recorded(path): + record_approval_request(path) + raise llm.PauseChain("waiting for approval to delete " + path) + do_delete(path) + return "deleted" +``` + +Unlike other exceptions - which are converted into `"Error: ..."` tool results and sent back to the model - `PauseChain` propagates out of the chain to your code. No provider call is made with a placeholder result. Before re-raising, the framework populates two attributes: + +- `pause.tool_call` - the `llm.ToolCall` whose implementation paused +- `pause.tool_results` - results of sibling calls in the same batch that completed + +```python +try: + chain_response.text() +except llm.PauseChain as pause: + print("Paused on", pause.tool_call.name, pause.tool_call.tool_call_id) +``` + +The failure semantics are defined: concurrent (async) sibling tool calls always run to completion before the exception propagates - their `after_call` hooks fire and their results are preserved - while sequential (sync) execution stops at the paused call, leaving later calls unexecuted so they can safely run on resume. If several concurrent calls pause, the first by call order propagates. `after_call` does not fire for a paused call, and no `ToolResult` is recorded for it - which is what marks it as still pending. + +(python-api-tools-resume)= + +#### Resuming a chain with pending tool calls + +To resume after a pause (or a crash, or a server restart), re-run the chain with a `messages=` history that ends in the unresolved tool calls. When the last assistant message in the history contains tool calls that have no matching results, the chain executes them first - through the normal `before_call`/`after_call` machinery - and then sends the results to the model: + +```python +chain = model.chain( + messages=persisted_messages, # ends in assistant tool calls with no results + tools=[delete_files], + system=system_prompt, +) +chain.text() +``` + +Calls that already have results in trailing tool-role messages are skipped, so a batch where some calls completed before the pause only re-executes the unresolved ones. A re-executed tool may raise `PauseChain` again - multi-step approval flows work by repeating the cycle. If a user or assistant message follows the tool calls in the history, the conversation has moved on and nothing is re-executed. + +Matching uses `tool_call_id` (always populated for newly-created tool calls); id-less calls from older persisted histories match results by name. You can also execute an explicit list of calls directly with `response.execute_tool_calls(tool_calls_list=[...])`. + (python-api-tools-attachments)= #### Tools can return attachments diff --git a/llm/__init__.py b/llm/__init__.py index 5dd52a273..52d3564ee 100644 --- a/llm/__init__.py +++ b/llm/__init__.py @@ -10,6 +10,7 @@ AsyncResponse, Attachment, CancelToolCall, + PauseChain, Conversation, EmbeddingModel, EmbeddingModelWithAliases, @@ -66,6 +67,7 @@ "ModelError", "NeedsKeyException", "Options", + "PauseChain", "Prompt", "Response", "schema_dsl", diff --git a/llm/models.py b/llm/models.py index 04658ff1e..d4e580543 100644 --- a/llm/models.py +++ b/llm/models.py @@ -365,6 +365,32 @@ class CancelToolCall(Exception): pass +class PauseChain(Exception): + """Raise inside a tool implementation to pause the chain. + + Unlike other exceptions - which are converted into error ToolResults + and sent back to the model - PauseChain propagates out of + ``execute_tool_calls()`` and ``chain()``. Before it is re-raised the + framework populates two attributes: + + - ``tool_call``: the ToolCall whose implementation paused + - ``tool_results``: ToolResults of sibling calls in the same batch + that completed + + Concurrent (async) sibling tool calls always run to completion + before the exception propagates; sequential (sync) execution stops + at the paused call, leaving later calls unexecuted so they can + safely run when the chain is resumed. Resume by re-running the + chain with a ``messages=`` history that ends in the unresolved tool + calls. + """ + + def __init__(self, *args): + super().__init__(*args) + self.tool_call: Optional["ToolCall"] = None + self.tool_results: List["ToolResult"] = [] + + @dataclass class Prompt: "The prompt being sent to the model." @@ -1787,9 +1813,18 @@ def execute_tool_calls( *, before_call: Optional[BeforeCallSync] = None, after_call: Optional[AfterCallSync] = None, + tool_calls_list: Optional[List[ToolCall]] = None, ) -> List[ToolResult]: + """Execute tool calls using this response's tools. + + By default executes ``self.tool_calls()``; pass + ``tool_calls_list=`` to execute an explicit list instead (used + when resuming a chain whose history ends in unresolved calls). + """ tool_results = [] tools_by_name = {tool.name: tool for tool in self.prompt.tools} + if tool_calls_list is None: + tool_calls_list = self.tool_calls() # Run prepare() on all Toolbox instances that need it instances_to_prepare: list[Toolbox] = [] @@ -1802,7 +1837,7 @@ def execute_tool_calls( inst.prepare() inst._prepared = True - for tool_call in self.tool_calls(): + for tool_call in tool_calls_list: tool: Optional[Tool] = tools_by_name.get(tool_call.name) # Tool could be None if the tool was not found in the prompt tools, # but we still call the before_call method: @@ -1860,6 +1895,13 @@ def execute_tool_calls( if not isinstance(result, str): result = json.dumps(result, default=repr) + except PauseChain as ex: + # Pause: propagate instead of converting to an error + # result. Sequential execution stops here - later calls + # never started, so they can safely run on resume. + ex.tool_call = tool_call + ex.tool_results = list(tool_results) + raise except Exception as ex: result = f"Error: {ex}" exception = ex @@ -2111,8 +2153,16 @@ async def execute_tool_calls( *, before_call: Optional[BeforeCallAsync] = None, after_call: Optional[AfterCallAsync] = None, + tool_calls_list: Optional[List[ToolCall]] = None, ) -> List[ToolResult]: - tool_calls_list = await self.tool_calls() + """Execute tool calls using this response's tools. + + By default executes ``await self.tool_calls()``; pass + ``tool_calls_list=`` to execute an explicit list instead (used + when resuming a chain whose history ends in unresolved calls). + """ + if tool_calls_list is None: + tool_calls_list = await self.tool_calls() tools_by_name = {tool.name: tool for tool in self.prompt.tools} # Run async prepare_async() on all Toolbox instances that need it @@ -2130,6 +2180,13 @@ async def execute_tool_calls( indexed_results: List[tuple[int, ToolResult]] = [] async_tasks: List[asyncio.Task] = [] + async_task_indexes: List[int] = [] + # Defined failure semantics: a pause or error in one call must not + # orphan concurrently-running siblings. Pauses and hook failures + # are collected here and raised only after every task that was + # started has finished. + paused: List[tuple[int, PauseChain]] = [] + failures: List[tuple[int, BaseException]] = [] for idx, tc in enumerate(tool_calls_list): tool: Optional[Tool] = tools_by_name.get(tc.name) @@ -2173,6 +2230,11 @@ async def run_async(tc=tc, tool=tool, idx=idx): if isinstance(result, str) else json.dumps(result, default=repr) ) + except PauseChain as ex: + # Propagates out of the task; collected after + # the gather so siblings finish first. + ex.tool_call = tc + raise except Exception as ex: output = f"Error: {ex}" exception = ex @@ -2195,6 +2257,7 @@ async def run_async(tc=tc, tool=tool, idx=idx): return idx, tr async_tasks.append(asyncio.create_task(run_async())) + async_task_indexes.append(idx) else: # Sync implementation: do hooks and call inline @@ -2216,6 +2279,9 @@ async def run_async(tc=tc, tool=tool, idx=idx): ) ) continue + except Exception as ex: + failures.append((idx, ex)) + break exception = None attachments = [] @@ -2236,6 +2302,13 @@ async def run_async(tc=tc, tool=tool, idx=idx): if isinstance(res, str) else json.dumps(res, default=repr) ) + except PauseChain as ex: + # Inline execution stops here; later calls never + # start. Tasks already started are still awaited + # below before the pause propagates. + ex.tool_call = tc + paused.append((idx, ex)) + break except Exception as ex: output = f"Error: {ex}" exception = ex @@ -2249,20 +2322,46 @@ async def run_async(tc=tc, tool=tool, idx=idx): exception=exception, ) - if tool is not None and after_call: - cb2 = after_call(tool, tc, tr) - if inspect.isawaitable(cb2): - await cb2 + try: + if tool is not None and after_call: + cb2 = after_call(tool, tc, tr) + if inspect.isawaitable(cb2): + await cb2 + except Exception as ex: + failures.append((idx, ex)) + break indexed_results.append((idx, tr)) - # Await all async tasks in parallel + # Await every task that was started; return_exceptions so a pause + # or hook failure in one task cannot orphan its siblings mid-flight. if async_tasks: - indexed_results.extend(await asyncio.gather(*async_tasks)) + outcomes = await asyncio.gather(*async_tasks, return_exceptions=True) + for task_idx, outcome in zip(async_task_indexes, outcomes): + if isinstance(outcome, PauseChain): + paused.append((task_idx, outcome)) + elif isinstance(outcome, BaseException): + failures.append((task_idx, outcome)) + else: + indexed_results.append(outcome) # Reorder by original index indexed_results.sort(key=lambda x: x[0]) - return [tr for _, tr in indexed_results] + results = [tr for _, tr in indexed_results] + + # Hook failures are bugs: raise the first by call order. + if failures: + failures.sort(key=lambda item: item[0]) + raise failures[0][1] + + # Pauses propagate with the completed sibling results attached. + if paused: + paused.sort(key=lambda item: item[0]) + pause = paused[0][1] + pause.tool_results = results + raise pause + + return results def __aiter__(self): self._start = time.monotonic() @@ -2479,28 +2578,16 @@ def __repr__(self): return "".format(self.prompt.prompt, text) -def _chain_for_tool_results(prior_response, tool_results, attachments) -> List[Any]: - """Build the message chain for a tool-result turn in a chain loop. - - Takes the prior response's full input chain + its structured - output, then appends a tool-role message carrying the new - ToolResult outputs. Attachments (e.g. images returned by tools) - are folded into a subsequent user-role message. - - This is what gives ``response.prompt.messages`` on the tool- - result turn the complete history for the next provider call — - including any reasoning signatures or thoughtSignatures from the - prior turn. - """ +def _append_tool_results_to_chain(chain, tool_results, attachments) -> List[Any]: + """Append a tool-role message carrying ToolResults to a message + chain, plus a trailing user-role message for any attachments the + tools returned (mimics the legacy attachments=[] kwarg behavior).""" from .parts import ( AttachmentPart, Message, ToolResultPart, ) - chain: List[Any] = list(prior_response.prompt.messages) + list( - prior_response._messages_now() - ) if tool_results: chain.append( Message( @@ -2515,8 +2602,6 @@ def _chain_for_tool_results(prior_response, tool_results, attachments) -> List[A ], ) ) - # Attachments that came back from tools ride on a trailing user - # message (mimics the legacy attachments=[] kwarg behavior). if attachments: chain.append( Message( @@ -2527,6 +2612,86 @@ def _chain_for_tool_results(prior_response, tool_results, attachments) -> List[A return chain +def _chain_for_tool_results(prior_response, tool_results, attachments) -> List[Any]: + """Build the message chain for a tool-result turn in a chain loop. + + Takes the prior response's full input chain + its structured + output, then appends a tool-role message carrying the new + ToolResult outputs. + + This is what gives ``response.prompt.messages`` on the tool- + result turn the complete history for the next provider call — + including any reasoning signatures or thoughtSignatures from the + prior turn. + """ + chain: List[Any] = list(prior_response.prompt.messages) + list( + prior_response._messages_now() + ) + return _append_tool_results_to_chain(chain, tool_results, attachments) + + +def _trailing_pending_tool_calls(messages) -> List[ToolCall]: + """Find unresolved tool calls at the end of a message history. + + Returns ToolCall objects from the last assistant message containing + locally-executable tool_call parts, minus any that already have a + matching tool_result in subsequent tool-role messages. Returns [] + when the history has moved on past those calls (a user/assistant/ + system message follows them) - resuming only makes sense when the + calls are the latest thing that happened. + + Matching uses tool_call_id when present; id-less calls (histories + persisted before ids were guaranteed) match results by name, one + result consumed per call. + """ + from .parts import ToolCallPart, ToolResultPart + + last_index = None + call_parts: List[Any] = [] + for i, msg in enumerate(messages or []): + parts = getattr(msg, "parts", None) or [] + calls = [ + p for p in parts if isinstance(p, ToolCallPart) and not p.server_executed + ] + if getattr(msg, "role", None) == "assistant" and calls: + last_index = i + call_parts = calls + if last_index is None: + return [] + + results: List[Any] = [] + for msg in messages[last_index + 1 :]: + role = getattr(msg, "role", None) + if role == "tool": + results.extend( + p + for p in (getattr(msg, "parts", None) or []) + if isinstance(p, ToolResultPart) + ) + else: + # Conversation moved on past these calls + return [] + + matched_ids = {r.tool_call_id for r in results if r.tool_call_id} + unmatched_names = [r.name for r in results if not r.tool_call_id] + pending = [] + for part in call_parts: + if part.tool_call_id: + if part.tool_call_id in matched_ids: + continue + elif part.name in unmatched_names: + unmatched_names.remove(part.name) + continue + pending.append( + ToolCall( + name=part.name, + arguments=part.arguments or {}, + tool_call_id=part.tool_call_id, + ) + ) + return pending + + class _BaseChainResponse: prompt: "Prompt" stream: bool @@ -2564,6 +2729,41 @@ def log_to_db(self, db): assert False, "Should have been a Response or AsyncResponse" sync_response.log_to_db(db) + def _pending_tool_calls(self) -> List[ToolCall]: + """Unresolved tool calls at the end of this chain's history. + + Non-empty when the supplied messages= end in an assistant + message whose tool calls have no results yet - e.g. a chain + that paused on PauseChain and is being resumed from persisted + history.""" + if not self.prompt.tools: + return [] + return _trailing_pending_tool_calls(self.prompt.messages) + + def _resume_prompt(self, tool_results: List[ToolResult]) -> Prompt: + """The first prompt for a resumed chain: the original history + plus a tool-role message carrying the freshly-executed results - + the same shape as the chain loop's own tool-result turns.""" + prompt = self.prompt + attachments = [] + for tool_result in tool_results: + attachments.extend(tool_result.attachments) + next_chain = _append_tool_results_to_chain( + list(prompt.messages), tool_results, attachments + ) + return Prompt( + "", + self.model, + tools=prompt.tools, + tool_results=tool_results, + messages=next_chain, + system=prompt._system, + system_fragments=prompt.system_fragments, + options=prompt.options, + attachments=attachments, + hide_reasoning=prompt.hide_reasoning, + ) + class ChainResponse(_BaseChainResponse): _responses: List["Response"] @@ -2573,13 +2773,33 @@ class ChainResponse(_BaseChainResponse): def responses(self) -> Iterator[Response]: prompt = self.prompt count = 0 - current_response: Optional[Response] = Response( + initial_response = Response( prompt, self.model, self.stream, key=self._key, conversation=self.conversation, ) + # Resume: a history ending in unresolved tool calls means a + # previous run stopped (paused or crashed) before executing + # them. Execute those calls first - through the normal + # before_call/after_call machinery - then start the loop on + # the tool-result turn. This could raise llm.PauseChain. + pending_tool_calls = self._pending_tool_calls() + if pending_tool_calls: + tool_results = initial_response.execute_tool_calls( + before_call=self.before_call, + after_call=self.after_call, + tool_calls_list=pending_tool_calls, + ) + initial_response = Response( + self._resume_prompt(tool_results), + self.model, + self.stream, + key=self._key, + conversation=self.conversation, + ) + current_response: Optional[Response] = initial_response while current_response: count += 1 yield current_response @@ -2650,13 +2870,31 @@ class AsyncChainResponse(_BaseChainResponse): async def responses(self) -> AsyncIterator[AsyncResponse]: prompt = self.prompt count = 0 - current_response: Optional[AsyncResponse] = AsyncResponse( + initial_response = AsyncResponse( prompt, self.model, self.stream, key=self._key, conversation=self.conversation, ) + # Resume: see ChainResponse.responses() - execute trailing + # unresolved tool calls before the first provider call. This + # could raise llm.PauseChain. + pending_tool_calls = self._pending_tool_calls() + if pending_tool_calls: + tool_results = await initial_response.execute_tool_calls( + before_call=self.before_call, + after_call=self.after_call, + tool_calls_list=pending_tool_calls, + ) + initial_response = AsyncResponse( + self._resume_prompt(tool_results), + self.model, + self.stream, + key=self._key, + conversation=self.conversation, + ) + current_response: Optional[AsyncResponse] = initial_response while current_response: count += 1 yield current_response diff --git a/tests/test_pause_resume.py b/tests/test_pause_resume.py new file mode 100644 index 000000000..25bb37a24 --- /dev/null +++ b/tests/test_pause_resume.py @@ -0,0 +1,391 @@ +"""Tests for llm.PauseChain and chain resume from message history.""" + +import asyncio +import json + +import pytest + +import llm +from llm.parts import Message, TextPart, ToolCallPart, ToolResultPart + +# ---- PauseChain ---- + + +def test_pause_chain_sync_model(): + after_calls = [] + + def needs_input(path: str) -> str: + raise llm.PauseChain("waiting for approval") + + def before(tool, tool_call): + pass + + def after(tool, tool_call, tool_result): + after_calls.append(tool_result.name) + + model = llm.get_model("echo") + chain = model.chain( + json.dumps( + {"tool_calls": [{"name": "needs_input", "arguments": {"path": "/tmp"}}]} + ), + tools=[needs_input], + before_call=before, + after_call=after, + ) + with pytest.raises(llm.PauseChain) as exc_info: + chain.text() + + pause = exc_info.value + assert str(pause) == "waiting for approval" + assert pause.tool_call is not None + assert pause.tool_call.name == "needs_input" + assert pause.tool_call.arguments == {"path": "/tmp"} + assert pause.tool_call.tool_call_id.startswith("tc_") + assert pause.tool_results == [] + # after_call must not fire for the paused tool + assert after_calls == [] + # The response that requested the tool call completed normally + assert len(chain._responses) == 1 + + +@pytest.mark.asyncio +async def test_pause_chain_async_model_siblings_complete(): + after_calls = [] + executed = [] + + async def needs_input() -> str: + raise llm.PauseChain("hold on") + + async def sibling() -> str: + await asyncio.sleep(0.01) + executed.append("sibling") + return "done" + + async def after(tool, tool_call, tool_result): + after_calls.append(tool_result.name) + + model = llm.get_async_model("echo") + chain = model.chain( + json.dumps({"tool_calls": [{"name": "needs_input"}, {"name": "sibling"}]}), + tools=[needs_input, sibling], + after_call=after, + ) + with pytest.raises(llm.PauseChain) as exc_info: + await chain.text() + + pause = exc_info.value + assert pause.tool_call.name == "needs_input" + # The concurrent sibling ran to completion - no orphaned tasks + assert executed == ["sibling"] + assert after_calls == ["sibling"] + # Completed sibling results ride on the exception + assert [r.name for r in pause.tool_results] == ["sibling"] + assert pause.tool_results[0].output == "done" + + +def test_pause_chain_sync_model_stops_remaining_calls(): + executed = [] + + def pauser() -> str: + raise llm.PauseChain("wait") + + def later() -> str: + executed.append("later") + return "x" + + model = llm.get_model("echo") + chain = model.chain( + json.dumps({"tool_calls": [{"name": "pauser"}, {"name": "later"}]}), + tools=[pauser, later], + ) + with pytest.raises(llm.PauseChain) as exc_info: + chain.text() + # Sequential execution stops at the pause; later call never starts, + # so it can safely re-execute on resume. + assert executed == [] + assert exc_info.value.tool_results == [] + + +@pytest.mark.asyncio +async def test_pause_chain_async_first_of_two_pauses_propagates(): + async def pause_a() -> str: + raise llm.PauseChain("a") + + async def pause_b() -> str: + raise llm.PauseChain("b") + + model = llm.get_async_model("echo") + chain = model.chain( + json.dumps({"tool_calls": [{"name": "pause_a"}, {"name": "pause_b"}]}), + tools=[pause_a, pause_b], + ) + with pytest.raises(llm.PauseChain) as exc_info: + await chain.text() + assert str(exc_info.value) == "a" + assert exc_info.value.tool_call.name == "pause_a" + + +@pytest.mark.asyncio +async def test_async_hook_exception_does_not_orphan_siblings(): + """Defined failure semantics: an exception raised by an after_call + hook propagates only after all concurrent tool tasks finish.""" + executed = [] + + async def boomer() -> str: + return "boom" + + async def slow() -> str: + await asyncio.sleep(0.05) + executed.append("slow") + return "ok" + + async def after(tool, tool_call, tool_result): + if tool_result.name == "boomer": + raise ValueError("hook bug") + + model = llm.get_async_model("echo") + chain = model.chain( + json.dumps({"tool_calls": [{"name": "boomer"}, {"name": "slow"}]}), + tools=[boomer, slow], + after_call=after, + ) + with pytest.raises(ValueError, match="hook bug"): + await chain.text() + # The slow sibling was not orphaned mid-flight + assert executed == ["slow"] + + +@pytest.mark.asyncio +async def test_pause_chain_async_model_sync_tool(): + def pauser() -> str: + raise llm.PauseChain("wait") + + model = llm.get_async_model("echo") + chain = model.chain( + json.dumps({"tool_calls": [{"name": "pauser"}]}), + tools=[pauser], + ) + with pytest.raises(llm.PauseChain) as exc_info: + await chain.text() + assert exc_info.value.tool_call.name == "pauser" + + +# ---- chain resume from message history ---- + + +def _pending_history(tool_call_id="tc_resume1"): + return [ + Message(role="user", parts=[TextPart(text="Convert hello to uppercase")]), + Message( + role="assistant", + parts=[ + ToolCallPart( + name="upper", + arguments={"text": "hello"}, + tool_call_id=tool_call_id, + ) + ], + ), + ] + + +def test_chain_resumes_trailing_pending_tool_calls(): + executed = [] + hook_calls = [] + + def upper(text: str) -> str: + executed.append(text) + return text.upper() + + def before(tool, tool_call): + hook_calls.append(("before", tool_call.name, tool_call.tool_call_id)) + + def after(tool, tool_call, tool_result): + hook_calls.append(("after", tool_result.name, tool_result.tool_call_id)) + + model = llm.get_model("echo") + chain = model.chain( + None, + messages=_pending_history(), + tools=[upper], + before_call=before, + after_call=after, + ) + output = chain.text() + + # The pending call executed through the normal hook machinery + assert executed == ["hello"] + assert hook_calls == [ + ("before", "upper", "tc_resume1"), + ("after", "upper", "tc_resume1"), + ] + # The model then received the tool result (echo renders + # prompt.tool_results), correlated by the original id + data = json.loads(output) + assert data["tool_results"] == [ + {"name": "upper", "output": "HELLO", "tool_call_id": "tc_resume1"} + ] + # Exactly one provider call was made + assert len(chain._responses) == 1 + + +@pytest.mark.asyncio +async def test_chain_resumes_trailing_pending_tool_calls_async(): + executed = [] + + async def upper(text: str) -> str: + executed.append(text) + return text.upper() + + model = llm.get_async_model("echo") + chain = model.chain(None, messages=_pending_history(), tools=[upper]) + output = await chain.text() + + assert executed == ["hello"] + data = json.loads(output) + assert data["tool_results"] == [ + {"name": "upper", "output": "HELLO", "tool_call_id": "tc_resume1"} + ] + + +def test_resume_skips_calls_that_already_have_results(): + executed = [] + + def first() -> str: + executed.append("first") + return "one" + + def second() -> str: + executed.append("second") + return "two" + + history = [ + Message(role="user", parts=[TextPart(text="go")]), + Message( + role="assistant", + parts=[ + ToolCallPart(name="first", arguments={}, tool_call_id="tc_a"), + ToolCallPart(name="second", arguments={}, tool_call_id="tc_b"), + ], + ), + Message( + role="tool", + parts=[ToolResultPart(name="first", output="one", tool_call_id="tc_a")], + ), + ] + model = llm.get_model("echo") + chain = model.chain(None, messages=history, tools=[first, second]) + output = chain.text() + + assert executed == ["second"] + data = json.loads(output) + assert data["tool_results"] == [ + {"name": "second", "output": "two", "tool_call_id": "tc_b"} + ] + + +def test_no_resume_when_conversation_moved_on(): + executed = [] + + def upper(text: str) -> str: + executed.append(text) + return text.upper() + + history = _pending_history() + [ + Message(role="user", parts=[TextPart(text="never mind")]), + ] + model = llm.get_model("echo") + chain = model.chain(None, messages=history, tools=[upper]) + chain.text() + assert executed == [] + + +def test_no_resume_without_tools(): + model = llm.get_model("echo") + chain = model.chain(None, messages=_pending_history()) + # No tools provided: nothing to execute, chain proceeds normally + output = chain.text() + assert "tool_results" not in json.loads(output) + + +def test_resume_matches_idless_calls_by_name(): + # Histories persisted before guaranteed ids may have None ids + executed = [] + + def upper(text: str) -> str: + executed.append(text) + return text.upper() + + history = [ + Message(role="user", parts=[TextPart(text="go")]), + Message( + role="assistant", + parts=[ + ToolCallPart(name="upper", arguments={"text": "a"}, tool_call_id=None), + ToolCallPart(name="upper", arguments={"text": "b"}, tool_call_id=None), + ], + ), + Message( + role="tool", + parts=[ToolResultPart(name="upper", output="A", tool_call_id=None)], + ), + ] + model = llm.get_model("echo") + chain = model.chain(None, messages=history, tools=[upper]) + chain.text() + # One result already present: only one of the two calls re-executes + assert executed == ["b"] + + +def test_resume_ignores_server_executed_calls(): + executed = [] + + def upper(text: str) -> str: + executed.append(text) + return text.upper() + + history = [ + Message(role="user", parts=[TextPart(text="go")]), + Message( + role="assistant", + parts=[ + ToolCallPart( + name="upper", + arguments={"text": "x"}, + tool_call_id="tc_srv", + server_executed=True, + ) + ], + ), + ] + model = llm.get_model("echo") + chain = model.chain(None, messages=history, tools=[upper]) + chain.text() + assert executed == [] + + +def test_resumed_tool_can_pause_again(): + def needs_more(text: str) -> str: + raise llm.PauseChain("second question") + + history = [ + Message(role="user", parts=[TextPart(text="go")]), + Message( + role="assistant", + parts=[ + ToolCallPart( + name="needs_more", + arguments={"text": "x"}, + tool_call_id="tc_again", + ) + ], + ), + ] + model = llm.get_model("echo") + chain = model.chain(None, messages=history, tools=[needs_more]) + with pytest.raises(llm.PauseChain) as exc_info: + chain.text() + assert exc_info.value.tool_call.name == "needs_more" + assert exc_info.value.tool_call.tool_call_id == "tc_again" + # No provider call was made: the chain paused before reaching the model + assert len(chain._responses) == 0 From 92a9ca7cbf0c5abc5f8f6c5aca287e115108b040 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 Jun 2026 15:17:37 -0700 Subject: [PATCH 093/258] Async tool calls to missing tools now produce error results (#1483) The async execute_tool_calls() silently dropped calls to tools that were not in tools= (or had no implementation): output and exception were assigned but no ToolResult was ever appended, so the next provider call carried an assistant tool_call with no matching result - which OpenAI and Anthropic reject. The sync executor already returned an 'Error: tool ... does not exist' result. The async path now mirrors the sync one: before_call fires (and can CancelToolCall) even though the tool is unavailable, and an error ToolResult is appended in call order. Also removes the now-unreachable tool-is-None branch from the inline sync-implementation path. This matters more since chain resume landed: a pending call whose tool is no longer registered would otherwise never resolve. Co-authored-by: Claude Fable 5 --- llm/models.py | 129 +++++++++++++++++++++++++++----------------- tests/test_tools.py | 76 ++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 50 deletions(-) diff --git a/llm/models.py b/llm/models.py index d4e580543..d9bab3a64 100644 --- a/llm/models.py +++ b/llm/models.py @@ -2192,13 +2192,48 @@ async def execute_tool_calls( tool: Optional[Tool] = tools_by_name.get(tc.name) exception: Optional[Exception] = None - if tool is None: - output = f'Error: tool "{tc.name}" does not exist' - exception = KeyError(tc.name) - elif not tool.implementation: - output = f'Error: tool "{tc.name}" has no implementation' - exception = KeyError(tc.name) - elif inspect.iscoroutinefunction(tool.implementation): + if tool is None or not tool.implementation: + # Mirror the sync executor: append an error ToolResult so + # the provider still receives a result for every tool + # call. before_call fires even though the tool is + # unavailable. + if before_call: + try: + cb = before_call(tool, tc) + if inspect.isawaitable(cb): + await cb + except CancelToolCall as ex: + indexed_results.append( + ( + idx, + ToolResult( + name=tc.name, + output="Cancelled: " + str(ex), + tool_call_id=tc.tool_call_id, + exception=ex, + ), + ) + ) + continue + except Exception as ex: + failures.append((idx, ex)) + break + reason = "does not exist" if tool is None else "has no implementation" + msg = 'tool "{}" {}'.format(tc.name, reason) + indexed_results.append( + ( + idx, + ToolResult( + name=tc.name, + output="Error: " + msg, + tool_call_id=tc.tool_call_id, + exception=KeyError(msg), + ), + ) + ) + continue + + if inspect.iscoroutinefunction(tool.implementation): async def run_async(tc=tc, tool=tool, idx=idx): # before_call inside the task @@ -2286,52 +2321,46 @@ async def run_async(tc=tc, tool=tool, idx=idx): exception = None attachments = [] - if tool is None: - output = f'Error: tool "{tc.name}" does not exist' - exception = KeyError(tc.name) - else: - try: - res = tool.implementation(**_implementation_arguments(tool, tc)) - if inspect.isawaitable(res): - res = await res - if isinstance(res, ToolOutput): - attachments.extend(res.attachments) - res = res.output - output = ( - res - if isinstance(res, str) - else json.dumps(res, default=repr) - ) - except PauseChain as ex: - # Inline execution stops here; later calls never - # start. Tasks already started are still awaited - # below before the pause propagates. - ex.tool_call = tc - paused.append((idx, ex)) - break - except Exception as ex: - output = f"Error: {ex}" - exception = ex - - tr = ToolResult( - name=tc.name, - output=output, - attachments=attachments, - tool_call_id=tc.tool_call_id, - instance=_get_instance(tool.implementation), - exception=exception, + try: + res = tool.implementation(**_implementation_arguments(tool, tc)) + if inspect.isawaitable(res): + res = await res + if isinstance(res, ToolOutput): + attachments.extend(res.attachments) + res = res.output + output = ( + res if isinstance(res, str) else json.dumps(res, default=repr) ) + except PauseChain as ex: + # Inline execution stops here; later calls never + # start. Tasks already started are still awaited + # below before the pause propagates. + ex.tool_call = tc + paused.append((idx, ex)) + break + except Exception as ex: + output = f"Error: {ex}" + exception = ex + + tr = ToolResult( + name=tc.name, + output=output, + attachments=attachments, + tool_call_id=tc.tool_call_id, + instance=_get_instance(tool.implementation), + exception=exception, + ) - try: - if tool is not None and after_call: - cb2 = after_call(tool, tc, tr) - if inspect.isawaitable(cb2): - await cb2 - except Exception as ex: - failures.append((idx, ex)) - break + try: + if after_call: + cb2 = after_call(tool, tc, tr) + if inspect.isawaitable(cb2): + await cb2 + except Exception as ex: + failures.append((idx, ex)) + break - indexed_results.append((idx, tr)) + indexed_results.append((idx, tr)) # Await every task that was started; return_exceptions so a pause # or hook failure in one task cannot orphan its siblings mid-flight. diff --git a/tests/test_tools.py b/tests/test_tools.py index 919aec071..5f5388ca9 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -820,3 +820,79 @@ async def before(tool, tool_call): await chain_response.text() assert len(seen) == 1 assert seen[0] is not None and seen[0].startswith("tc_") + + +@pytest.mark.asyncio +async def test_async_missing_tool_produces_error_result(): + # Async executor parity with sync: a call to a tool that is not in + # tools= must produce an error ToolResult, not silently vanish - + # otherwise the next provider call has a tool_call with no result. + before_calls = [] + + async def real_tool() -> str: + return "ok" + + async def before(tool, tool_call): + # before_call fires even when tool is None, like the sync path + before_calls.append((tool.name if tool else None, tool_call.name)) + + model = llm.get_async_model("echo") + chain_response = model.chain( + json.dumps({"tool_calls": [{"name": "missing_tool"}, {"name": "real_tool"}]}), + tools=[real_tool], + before_call=before, + ) + await chain_response.text() + + second = chain_response._responses[1] + results = [(r.name, r.output) for r in second.prompt.tool_results] + assert results == [ + ("missing_tool", 'Error: tool "missing_tool" does not exist'), + ("real_tool", "ok"), + ] + assert isinstance(second.prompt.tool_results[0].exception, KeyError) + assert (None, "missing_tool") in before_calls + + +@pytest.mark.asyncio +async def test_async_missing_tool_can_be_cancelled_by_before_call(): + async def real_tool() -> str: + return "ok" + + async def before(tool, tool_call): + if tool is None: + raise CancelToolCall("no such tool") + + model = llm.get_async_model("echo") + chain_response = model.chain( + json.dumps({"tool_calls": [{"name": "missing_tool"}, {"name": "real_tool"}]}), + tools=[real_tool], + before_call=before, + ) + await chain_response.text() + second = chain_response._responses[1] + results = [(r.name, r.output) for r in second.prompt.tool_results] + assert results == [ + ("missing_tool", "Cancelled: no such tool"), + ("real_tool", "ok"), + ] + + +@pytest.mark.asyncio +async def test_async_tool_without_implementation_produces_error_result(): + tool = llm.Tool( + name="no_impl", + description="A tool with no implementation", + input_schema={"type": "object", "properties": {}}, + implementation=None, + ) + model = llm.get_async_model("echo") + chain_response = model.chain( + json.dumps({"tool_calls": [{"name": "no_impl"}]}), + tools=[tool], + ) + await chain_response.text() + second = chain_response._responses[1] + assert [(r.name, r.output) for r in second.prompt.tool_results] == [ + ("no_impl", 'Error: tool "no_impl" has no implementation'), + ] From 2ae70895082b8a4158e757fbdc831c9aceffbda6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 Jun 2026 15:25:01 -0700 Subject: [PATCH 094/258] Release 0.32a3 Refs #1480, #1481, #1482, #1482, #1482, #1483 --- docs/changelog.md | 12 ++++++++++++ pyproject.toml | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 40aefd2fe..ab24168fb 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,17 @@ # Changelog +(v0_32_a3)= +## 0.32a3 (2026-06-09) + +Driven by the needs of [datasette-agent](https://github.com/datasette/datasette-agent)'s human-in-the-loop `ask_user()` feature, made the following improvements to how tool calls work: + +- Tool implementations can declare a parameter named `llm_tool_call` in order to be passed the `llm.ToolCall` object for the current invocation. This allows them to access the current `llm_tool_call.tool_call_id`. See {ref}`python-api-tools-llm-tool-call`. [#1480](https://github.com/simonw/llm/pull/1480) +- Every tool call is now guaranteed a unique `tool_call_id` - providers that do not supply one get a synthesized `tc_`-prefixed ULID. [#1481](https://github.com/simonw/llm/pull/1481) +- Tools can raise a `llm.PauseChain` exception to cleanly pause the tool chain, useful for things like waiting for human approval. The exception propagates to the caller with `.tool_call` and `.tool_results` (completed sibling results) attached, and no model call is made with a placeholder result. See {ref}`python-api-tools-pause`. [#1482](https://github.com/simonw/llm/pull/1482) +- Failure semantics for concurrent tool execution: async sibling tool calls always run to completion before a pause or hook exception propagates. [#1482](https://github.com/simonw/llm/pull/1482) +- Chains can now resume from a `messages=` history ending in unresolved tool calls: the calls are executed through the normal `before_call`/`after_call` machinery before the first model call, skipping any that already have results. The `execute_tool_calls()` method also accepts a new optional `tool_calls_list=` argument for executing an explicit list of `ToolCall` objects in place of the calls requested by the response. See {ref}`python-api-tools-resume`. [#1482](https://github.com/simonw/llm/pull/1482) +- Fixed a bug where the async tool executor silently dropped calls to tools not present in `tools=` - these now return `Error: tool "..." does not exist` results, matching the sync executor. [#1483](https://github.com/simonw/llm/pull/1483) + (v0_32_a2)= ## 0.32a2 (2026-05-12) diff --git a/pyproject.toml b/pyproject.toml index 9725cdfa7..0241119a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm" -version = "0.32a2" +version = "0.32a3" description = "CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine." readme = { file = "README.md", content-type = "text/markdown" } authors = [ From 1e72f0a0e4dd7950d8580862dd38a3c148823c04 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 Jun 2026 15:25:37 -0700 Subject: [PATCH 095/258] Spelling for Datasette Agent --- docs/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index ab24168fb..b152f27e9 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,7 +3,7 @@ (v0_32_a3)= ## 0.32a3 (2026-06-09) -Driven by the needs of [datasette-agent](https://github.com/datasette/datasette-agent)'s human-in-the-loop `ask_user()` feature, made the following improvements to how tool calls work: +Driven by the needs of [Datasette Agent](https://github.com/datasette/datasette-agent)'s human-in-the-loop `ask_user()` feature, made the following improvements to how tool calls work: - Tool implementations can declare a parameter named `llm_tool_call` in order to be passed the `llm.ToolCall` object for the current invocation. This allows them to access the current `llm_tool_call.tool_call_id`. See {ref}`python-api-tools-llm-tool-call`. [#1480](https://github.com/simonw/llm/pull/1480) - Every tool call is now guaranteed a unique `tool_call_id` - providers that do not supply one get a synthesized `tc_`-prefixed ULID. [#1481](https://github.com/simonw/llm/pull/1481) From 5fe711964e1ecf5cc195aa47d5bd51b968078541 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 Jun 2026 15:31:54 -0700 Subject: [PATCH 096/258] Update docs/fragments.md --- docs/fragments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/fragments.md b/docs/fragments.md index 318a893f5..ac29b7235 100644 --- a/docs/fragments.md +++ b/docs/fragments.md @@ -30,7 +30,7 @@ from importlib.metadata import version llm_version = version("llm") cog.out(f'The URL will be fetched with the user-agent `llm/{llm_version} (https://llm.datasette.io/)`.') ]]]--> -The URL will be fetched with the user-agent `llm/0.32a2 (https://llm.datasette.io/)`. +The URL will be fetched with the user-agent `llm/0.32a3 (https://llm.datasette.io/)`. The `-f` option can be used multiple times to combine together multiple fragments. From d492388220b3d133b0bf9d248780b2fafd768672 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 Jun 2026 15:32:08 -0700 Subject: [PATCH 097/258] Upgrade GitHub Actions --- .github/workflows/cog.yml | 4 ++-- .github/workflows/publish.yml | 12 ++++++------ .github/workflows/stable-docs.yml | 2 +- .github/workflows/test.yml | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cog.yml b/.github/workflows/cog.yml index d46c0f3b0..6ff5df5f2 100644 --- a/.github/workflows/cog.yml +++ b/.github/workflows/cog.yml @@ -13,12 +13,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: ref: ${{ github.head_ref }} - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.11' diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ab73b83af..40e4cee24 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,13 +14,13 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | pip install . --group dev @@ -34,13 +34,13 @@ jobs: id-token: write needs: [test] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.13' cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | pip install setuptools wheel build diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml index 84b829e96..f55ac85ff 100644 --- a/.github/workflows/stable-docs.yml +++ b/.github/workflows/stable-docs.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: fetch-depth: 0 # We need all commits to find docs/ changes - name: Set up Git user diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0c6fccaae..88ca340f2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,13 +13,13 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | pip install . --group dev @@ -54,7 +54,7 @@ jobs: ./tests/test-llm-load-plugins.sh - name: Upload artifact of builds if: matrix.python-version == '3.13' && matrix.os == 'ubuntu-latest' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: dist-${{ matrix.os }}-${{ matrix.python-version }} path: dist/* From 0d593ea2a45d81cb622ded87ebd5ed47eeb6a263 Mon Sep 17 00:00:00 2001 From: Niall Smart Date: Tue, 9 Jun 2026 18:47:25 -0400 Subject: [PATCH 098/258] Include system prompt in pre-baked chain messages Closes #1478 --- llm/models.py | 34 ++++++++++++++++++++++++++++------ tests/test_parts.py | 25 +++++++++++++++---------- 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/llm/models.py b/llm/models.py index d9bab3a64..80fb6b1a2 100644 --- a/llm/models.py +++ b/llm/models.py @@ -451,12 +451,7 @@ def prompt(self): @property def system(self): "The system prompt, with any system fragments concatenated." - bits = [ - bit.strip() - for bit in (self.system_fragments + [self._system or ""]) - if bit.strip() - ] - return "\n\n".join(bits) + return _combine_system(self._system, self.system_fragments) @property def messages(self): @@ -536,6 +531,16 @@ def _wrap_tools(tools: List[ToolDef]) -> List[Tool]: return wrapped_tools +def _combine_system(system, system_fragments): + "Concatenate the system prompt and any system fragments into one string." + bits = [ + bit.strip() + for bit in ((system_fragments or []) + [system or ""]) + if bit.strip() + ] + return "\n\n".join(bits) + + def _merge_options(options: Optional[dict], kwargs: dict) -> dict: if not options: return kwargs @@ -568,6 +573,8 @@ def _build_full_chain( attachments, tool_results, explicit_messages, + system=None, + system_fragments=None, ) -> List[Any]: """Build the full message chain for the next turn. @@ -600,6 +607,13 @@ def _build_full_chain( # append that response's structured output. chain.extend(last.prompt.messages) chain.extend(last._messages_now()) + else: + # Start with the system prompt as the first message so adapters + # that build from prompt.messages see it. On later turns it + # is already carried forward in last.prompt.messages. + system_text = _combine_system(system, system_fragments) + if system_text: + chain.append(Message(role="system", parts=[TextPart(text=system_text)])) # Append the new turn's input if tool_results: @@ -659,6 +673,8 @@ def prompt( attachments=attachments, tool_results=tool_results, explicit_messages=messages, + system=system, + system_fragments=system_fragments, ) return Response( Prompt( @@ -711,6 +727,8 @@ def chain( attachments=attachments, tool_results=tool_results, explicit_messages=messages, + system=system, + system_fragments=system_fragments, ) return ChainResponse( Prompt( @@ -783,6 +801,8 @@ def chain( attachments=attachments, tool_results=tool_results, explicit_messages=messages, + system=system, + system_fragments=system_fragments, ) return AsyncChainResponse( Prompt( @@ -832,6 +852,8 @@ def prompt( attachments=attachments, tool_results=tool_results, explicit_messages=messages, + system=system, + system_fragments=system_fragments, ) return AsyncResponse( Prompt( diff --git a/tests/test_parts.py b/tests/test_parts.py index f2a03e315..f2a89bc75 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -1688,6 +1688,12 @@ class TestChainPropagatesSystem: adapters that read prompt.system (OpenAI and other stateless-per-turn providers) see it on every call.""" + def assert_system(self, prompt, *expected): + assert prompt.messages[0].role == "system" + for e in expected: + assert e in prompt.system + assert e in prompt.messages[0].parts[0].text + def test_sync_chain_tool_result_turn_preserves_system(self, mock_model): # First turn: fake a tool call so the chain iterates. tool_call = llm.ToolCall(tool_call_id="c1", name="tick", arguments={}) @@ -1714,8 +1720,7 @@ def tick() -> str: chain = m.chain("q", system="be brief", tools=[tick]) list(chain.responses()) # Second response was the tool-result turn. - second = chain._responses[1] - assert second.prompt.system == "be brief" + self.assert_system(chain._responses[1].prompt, "be brief") def test_sync_chain_tool_result_turn_preserves_system_fragments(self, mock_model): tool_call = llm.ToolCall(tool_call_id="c1", name="tick", arguments={}) @@ -1746,12 +1751,9 @@ def tick() -> str: tools=[tick], ) list(chain.responses()) - second = chain._responses[1] - # prompt.system concatenates _system + system_fragments; all - # three strings should be preserved on the tool-result turn. - assert "inline sys" in second.prompt.system - assert "fragment A" in second.prompt.system - assert "fragment B" in second.prompt.system + self.assert_system( + chain._responses[1].prompt, "inline sys", "fragment A", "fragment B" + ) @pytest.mark.asyncio async def test_async_chain_tool_result_turn_preserves_system( @@ -1784,8 +1786,11 @@ def tick() -> str: responses = [] async for r in chain.responses(): responses.append(r) - second = chain._responses[1] - assert second.prompt.system == "be brief" + self.assert_system(responses[1].prompt, "be brief") + + def test_chain_includes_system_in_messages(self, mock_model): + chain = mock_model.chain("q", system="be brief") + self.assert_system(chain.prompt, "be brief") # chain() accepts messages= (parity with prompt()) From b7e213282d017bb96f9dc06d48b372532f4cf0de Mon Sep 17 00:00:00 2001 From: Eldar Shlomi <72104254+eldar702@users.noreply.github.com> Date: Mon, 15 Jun 2026 20:00:48 +0300 Subject: [PATCH 099/258] docs: fix wrong hook name and typo in register_fragment_loaders section (#1489) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The description of the `register_fragment_loaders` hook incorrectly referred to `register_template_loaders` as the hook to use. These are two distinct hooks; fragment loader plugins must use `register_fragment_loaders`. Also fixed a typo: "de-duplicatino" → "de-duplication". AI-assisted contribution. --- docs/plugins/plugin-hooks.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/plugin-hooks.md b/docs/plugins/plugin-hooks.md index 7adc5fe01..edd32837d 100644 --- a/docs/plugins/plugin-hooks.md +++ b/docs/plugins/plugin-hooks.md @@ -235,7 +235,7 @@ Note that `functions:` provided by templates using this plugin hook will not be (plugin-hooks-register-fragment-loaders)= ## register_fragment_loaders(register) -Plugins can register new fragment loaders using the `register_template_loaders` hook. These can then be used with the `llm -f prefix:argument` syntax. +Plugins can register new fragment loaders using the `register_fragment_loaders` hook. These can then be used with the `llm -f prefix:argument` syntax. Fragment loader plugins differ from template loader plugins in that you can stack more than one fragment loader call together in the same prompt. @@ -289,4 +289,4 @@ llm -f my-fragments:argument ``` If multiple fragments are returned they will be used as if the user passed multiple `-f X` arguments to the command. -Multiple fragments are particularly useful for things like plugins that return every file in a directory. If these were concatenated together by the plugin, a change to a single file would invalidate the de-duplicatino cache for that whole fragment. Giving each file its own fragment means we can avoid storing multiple copies of that full collection if only a single file has changed. +Multiple fragments are particularly useful for things like plugins that return every file in a directory. If these were concatenated together by the plugin, a change to a single file would invalidate the de-duplication cache for that whole fragment. Giving each file its own fragment means we can avoid storing multiple copies of that full collection if only a single file has changed. From af977da5ca2e8be0056dd3d909740954f729de4f Mon Sep 17 00:00:00 2001 From: Labib Bin Salam <98468420+Labib-Bin-Salam@users.noreply.github.com> Date: Mon, 15 Jun 2026 18:01:40 +0100 Subject: [PATCH 100/258] Fix typos in documentation (#1486) - tools.md: drop the stray article in "every tool is a defined as" and the doubled word in "Toolbox classes can be be configured". - contributing.md: fix the doubled "an an" in the example prompt (both the streaming and --no-stream snippets). --- docs/contributing.md | 4 ++-- docs/tools.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/contributing.md b/docs/contributing.md index 3459258fc..63e2e4ec5 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -27,7 +27,7 @@ The default OpenAI plugin has a debugging mechanism for showing the exact reques Set the `LLM_OPENAI_SHOW_RESPONSES` environment variable like this: ```bash -LLM_OPENAI_SHOW_RESPONSES=1 uv run llm -m chatgpt 'three word slogan for an an otter-run bakery' +LLM_OPENAI_SHOW_RESPONSES=1 uv run llm -m chatgpt 'three word slogan for an otter-run bakery' ``` This will output details of the API requests and responses to the console. @@ -35,7 +35,7 @@ Use `--no-stream` to see a more readable version of the body that avoids streami ```bash LLM_OPENAI_SHOW_RESPONSES=1 uv run llm -m chatgpt --no-stream \ - 'three word slogan for an an otter-run bakery' + 'three word slogan for an otter-run bakery' ``` ## Documentation diff --git a/docs/tools.md b/docs/tools.md index 945ae8dbd..bc37078b8 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -64,11 +64,11 @@ Further tools can be installed using plugins, or you can use the `llm --function ## LLM's implementation of tools -In LLM every tool is a defined as a Python function. The function can take any number of arguments and can return a string or an object that can be converted to a string. +In LLM every tool is defined as a Python function. The function can take any number of arguments and can return a string or an object that can be converted to a string. Tool functions should include a docstring that describes what the function does. This docstring will become the description that is passed to the model. -Tools can also be defined as {ref}`toolbox classes `, a subclass of `llm.Toolbox` that allows multiple related tools to be bundled together. Toolbox classes can be be configured when they are instantiated, and can also maintain state in between multiple tool calls. +Tools can also be defined as {ref}`toolbox classes `, a subclass of `llm.Toolbox` that allows multiple related tools to be bundled together. Toolbox classes can be configured when they are instantiated, and can also maintain state in between multiple tool calls. The Python API can accept functions directly. The command-line interface has two ways for tools to be defined: via plugins that implement the {ref}`register_tools() plugin hook `, or directly on the command-line using the `--functions` argument to specify a block of Python code defining one or more functions - or a path to a Python file containing the same. From 4da36ff523e19f53925fd40605571b83243052a6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 16:49:22 -0700 Subject: [PATCH 101/258] Try sqlite-utils 4.0rc1 in CI Also output current sqlite-utils version in pytest headers Refs https://github.com/simonw/sqlite-utils/issues/758 --- .github/workflows/test.yml | 9 +++++++++ tests/conftest.py | 13 +++++++++++++ 2 files changed, 22 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88ca340f2..ef5d0d0f2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,6 +12,11 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + sqlite-utils-version: [""] + include: + - os: ubuntu-latest + python-version: "3.14" + sqlite-utils-version: "4.0rc1" steps: - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} @@ -23,6 +28,10 @@ jobs: - name: Install dependencies run: | pip install . --group dev + - name: Install sqlite-utils pre-release + if: matrix.sqlite-utils-version != '' + run: | + pip install sqlite-utils==${{ matrix.sqlite-utils-version }} - name: Run tests run: | python -m pytest -vv diff --git a/tests/conftest.py b/tests/conftest.py index f004ed095..f26b4e99c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ +import importlib.metadata import pytest import sqlite_utils import json +import sqlite3 import llm import llm_echo from llm.plugins import pm @@ -15,6 +17,17 @@ def pytest_configure(config): sys._called_from_test = True +def pytest_report_header(config): + conn = sqlite3.connect(":memory:") + version = conn.execute("select sqlite_version()").fetchone()[0] + conn.close() + sqlite_utils_version = importlib.metadata.version("sqlite-utils") + return [ + "SQLite: {}".format(version), + "sqlite-utils: {}".format(sqlite_utils_version), + ] + + @pytest.fixture def user_path(tmpdir): dir = tmpdir / "llm.datasette.io" From 6f2dba429abbd54f7ea8012d795ae7d5c3c4caff Mon Sep 17 00:00:00 2001 From: Mrmaxmeier <3913977+Mrmaxmeier@users.noreply.github.com> Date: Mon, 22 Jun 2026 02:08:40 +0200 Subject: [PATCH 102/258] Update click to >8.2.0 The was previously blocked by removed support of Python 3.9, but we have since dropped support as well: dd227cdcd18b3e26853d494f4d0115963b797a9d Closes #1024 --- pyproject.toml | 2 +- tests/test_cli_openai_models.py | 4 ++-- tests/test_embed_cli.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0241119a6..55b7b39db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ [dependency-groups] dev = [ "build", - "click<8.2.0", # https://github.com/simonw/llm/issues/1024 + "click>=8.2.0", "pytest", "numpy", "pytest-httpx>=0.33.0", diff --git a/tests/test_cli_openai_models.py b/tests/test_cli_openai_models.py index ac353eefb..85d753120 100644 --- a/tests/test_cli_openai_models.py +++ b/tests/test_cli_openai_models.py @@ -487,7 +487,7 @@ def test_gpt4o_mini_sync_and_async(monkeypatch, tmpdir, httpx_mock, async_, usag }, headers={"Content-Type": "application/json"}, ) - runner = CliRunner(mix_stderr=False) + runner = CliRunner() args = ["-m", "gpt-4o-mini", "--key", "x", "--no-stream"] if usage: args.append(usage) @@ -495,7 +495,7 @@ def test_gpt4o_mini_sync_and_async(monkeypatch, tmpdir, httpx_mock, async_, usag args.append("--async") result = runner.invoke(cli, args, catch_exceptions=False) assert result.exit_code == 0 - assert result.output == "Ho ho ho\n" + assert result.stdout == "Ho ho ho\n" if usage: assert result.stderr == "Token usage: 1,000 input, 2,000 output\n" # Confirm it was correctly logged diff --git a/tests/test_embed_cli.py b/tests/test_embed_cli.py index afee77122..c8e8b8ce6 100644 --- a/tests/test_embed_cli.py +++ b/tests/test_embed_cli.py @@ -582,7 +582,7 @@ def test_embed_multi_files_errors(multi_files, args, expected_error): ) def test_embed_multi_files_encoding(multi_files, extra_args, expected_error): db_path, files = multi_files - runner = CliRunner(mix_stderr=False) + runner = CliRunner() result = runner.invoke( cli, [ From 3c9608d0edff8eb1b0f8a6c1ee9da9504e47f1a0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 17:09:06 -0700 Subject: [PATCH 103/258] Only package llm This avoids problems if I have untracked directories in here. --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 55b7b39db..a5325681a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,9 @@ Changelog = "https://github.com/simonw/llm/releases" [project.scripts] llm = "llm.cli:cli" +[tool.setuptools.packages.find] +include = ["llm*"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" From 512659547241a61e30116e9ada4db34a624062ae Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 17:09:44 -0700 Subject: [PATCH 104/258] Revert "Only package llm" This reverts commit 3c9608d0edff8eb1b0f8a6c1ee9da9504e47f1a0. I had made this change already. --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a5325681a..55b7b39db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,9 +78,6 @@ Changelog = "https://github.com/simonw/llm/releases" [project.scripts] llm = "llm.cli:cli" -[tool.setuptools.packages.find] -include = ["llm*"] - [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" From a66a38f16e197af8b1979c11536ec7f4b6a97787 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 17:12:32 -0700 Subject: [PATCH 105/258] Apply pytest fix again, refs #1024 We had new code that post-dated the PR that fixed this. --- tests/test_cli_streaming.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_cli_streaming.py b/tests/test_cli_streaming.py index e83586881..4414706fc 100644 --- a/tests/test_cli_streaming.py +++ b/tests/test_cli_streaming.py @@ -11,7 +11,7 @@ def test_text_goes_to_stdout_not_stderr(mock_model): mock_model.enqueue(["Hello world"]) - runner = CliRunner(mix_stderr=False) + runner = CliRunner() result = runner.invoke( cli, ["-m", "mock", "hi", "--no-log"], @@ -32,7 +32,7 @@ def test_reasoning_goes_to_stderr_not_stdout(mock_model): llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), ] ) - runner = CliRunner(mix_stderr=False) + runner = CliRunner() result = runner.invoke( cli, ["-m", "mock", "hi", "--no-log"], @@ -52,7 +52,7 @@ def test_reasoning_rendered_in_dim_style(mock_model): llm.parts.StreamEvent(type="text", chunk="x", part_index=1), ] ) - runner = CliRunner(mix_stderr=False) + runner = CliRunner() result = runner.invoke( cli, ["-m", "mock", "hi", "--no-log"], @@ -74,7 +74,7 @@ def test_hide_reasoning_flag_suppresses_reasoning(mock_model): llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), ] ) - runner = CliRunner(mix_stderr=False) + runner = CliRunner() result = runner.invoke( cli, ["-m", "mock", "hi", "--no-log", "--hide-reasoning"], @@ -94,7 +94,7 @@ def test_hide_reasoning_short_flag_R(mock_model): llm.parts.StreamEvent(type="text", chunk="x", part_index=1), ] ) - runner = CliRunner(mix_stderr=False) + runner = CliRunner() result = runner.invoke( cli, ["-m", "mock", "hi", "--no-log", "-R"], @@ -113,7 +113,7 @@ def test_newline_between_reasoning_and_text(mock_model): llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), ] ) - runner = CliRunner(mix_stderr=False) + runner = CliRunner() result = runner.invoke( cli, ["-m", "mock", "hi", "--no-log"], @@ -133,7 +133,7 @@ def test_async_path_reasoning_to_stderr(async_mock_model): llm.parts.StreamEvent(type="text", chunk="async answer", part_index=1), ] ) - runner = CliRunner(mix_stderr=False) + runner = CliRunner() result = runner.invoke( cli, ["-m", "mock", "hi", "--async", "--no-log"], @@ -148,7 +148,7 @@ def test_plain_str_plugin_still_works(mock_model): """A plugin that yields plain strings (legacy) still displays correctly — no reasoning branch, everything to stdout.""" mock_model.enqueue(["plain ", "text"]) - runner = CliRunner(mix_stderr=False) + runner = CliRunner() result = runner.invoke( cli, ["-m", "mock", "hi", "--no-log"], From 94769b8b076cde9392059d76bd766453cf900180 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 17:19:59 -0700 Subject: [PATCH 106/258] Fix for test that fails with sqlite-utils 4.0rc1 Refs #https://github.com/simonw/sqlite-utils/issues/758#issuecomment-4763695884 --- tests/test_fragments_cli.py | 104 ++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/tests/test_fragments_cli.py b/tests/test_fragments_cli.py index 5975c9e70..28fd3b4e9 100644 --- a/tests/test_fragments_cli.py +++ b/tests/test_fragments_cli.py @@ -1,6 +1,7 @@ from click.testing import CliRunner from importlib.metadata import version from llm.cli import cli +from llm.migrations import migrate from unittest import mock import os import yaml @@ -69,61 +70,60 @@ def get_list(): def test_fragments_list(user_path): runner = CliRunner() - with runner.isolated_filesystem(): - # This is just to create the database schema - with open("fragment1.txt", "w") as f: - f.write("1") - assert ( - runner.invoke(cli, ["fragments", "set", "f1", "fragment1.txt"]).exit_code - == 0 + db = sqlite_utils.Database(str(user_path / "logs.db")) + with db.conn: + migrate(db) + db["fragments"].insert_all( + [ + { + "id": 1, + "content": "1", + "datetime_utc": "2023-10-01T00:00:00Z", + "source": "file1.txt", + "hash": "hash1", + }, + { + "id": 2, + "content": "2", + "datetime_utc": "2022-10-01T00:00:00Z", + "source": "file2.txt", + "hash": "hash2", + }, + { + "id": 3, + "content": "3", + "datetime_utc": "2024-10-01T00:00:00Z", + "source": "file3.txt", + "hash": "hash3", + }, + ] ) - # Now add the rest directly to the database - db = sqlite_utils.Database(str(user_path / "logs.db")) - db["fragments"].delete_where() - db["fragments"].insert( + db["fragment_aliases"].insert( { - "content": "1", - "datetime_utc": "2023-10-01T00:00:00Z", - "source": "file1.txt", - "hash": "hash1", - }, + "alias": "f1", + "fragment_id": 1, + } ) - db["fragments"].insert( - { - "content": "2", - "datetime_utc": "2022-10-01T00:00:00Z", - "source": "file2.txt", - "hash": "hash2", - }, - ) - db["fragments"].insert( - { - "content": "3", - "datetime_utc": "2024-10-01T00:00:00Z", - "source": "file3.txt", - "hash": "hash3", - }, - ) - result = runner.invoke(cli, ["fragments", "list"]) - assert result.exit_code == 0 - assert result.output.strip() == (textwrap.dedent(""" - - hash: hash2 - aliases: [] - datetime_utc: '2022-10-01T00:00:00Z' - source: file2.txt - content: '2' - - hash: hash1 - aliases: - - f1 - datetime_utc: '2023-10-01T00:00:00Z' - source: file1.txt - content: '1' - - hash: hash3 - aliases: [] - datetime_utc: '2024-10-01T00:00:00Z' - source: file3.txt - content: '3' - """).strip()) + result = runner.invoke(cli, ["fragments", "list"]) + assert result.exit_code == 0 + assert result.output.strip() == (textwrap.dedent(""" + - hash: hash2 + aliases: [] + datetime_utc: '2022-10-01T00:00:00Z' + source: file2.txt + content: '2' + - hash: hash1 + aliases: + - f1 + datetime_utc: '2023-10-01T00:00:00Z' + source: file1.txt + content: '1' + - hash: hash3 + aliases: [] + datetime_utc: '2024-10-01T00:00:00Z' + source: file3.txt + content: '3' + """).strip()) @mock.patch.dict(os.environ, {"OPENAI_API_KEY": "X"}) From 556c1eaa14f0289b9966f9cba90d2a63223510aa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 10:00:06 -0700 Subject: [PATCH 107/258] Dedupe tool descriptions in logs, better argument display, closes #1515 --- llm/cli.py | 69 +++++++++++++++++++++++++++++++------ tests/test_llm_logs.py | 77 ++++++++++++++++++++++++++++++++++++++++++ tests/test_tools.py | 4 ++- 3 files changed, 138 insertions(+), 12 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index 263207252..e91f24bab 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -2052,6 +2052,46 @@ def logs_list( click.echo(output) else: # Output neatly formatted human-readable logs + def _fenced_block(value): + # Fenced code block, indented to nest inside a list item + num_backticks = 3 + while "`" * num_backticks in value: + num_backticks += 1 + fence = "`" * num_backticks + return textwrap.indent("{}\n{}\n{}".format(fence, value, fence), " ") + + def _inline_code(value): + num_backticks = 1 + while "`" * num_backticks in value: + num_backticks += 1 + delimiter = "`" * num_backticks + if value.startswith("`") or value.endswith("`"): + return "{} {} {}".format(delimiter, value, delimiter) + return "{}{}{}".format(delimiter, value, delimiter) + + def _format_tool_call_arguments(arguments): + if not isinstance(arguments, dict) or not arguments: + return " Arguments: {}".format(_inline_code(json.dumps(arguments))) + lines = [] + for key, value in arguments.items(): + if isinstance(value, str): + lines.append(" {}:".format(key)) + lines.append(_fenced_block(value)) + else: + lines.append( + " {}: {}".format(key, _inline_code(json.dumps(value))) + ) + return "\n".join(lines) + + def _token_usage_markdown(input_tokens, output_tokens, token_details): + usage = token_usage_string(input_tokens, output_tokens, None) + if token_details: + details = _inline_code(json.dumps(token_details)) + if usage: + return "{}, {}".format(usage, details) + return details + return usage + def _display_fragments(fragments, title): if not fragments: return @@ -2073,6 +2113,7 @@ def _display_fragments(fragments, title): current_system = None should_show_conversation = True + seen_tool_hashes = set() for row in rows: if short: system = truncate_string( @@ -2184,14 +2225,20 @@ def _display_fragments(fragments, title): if row["tools"]: click.echo("\n### Tools\n") for tool in row["tools"]: - click.echo( - "- **{}**: `{}`
\n {}
\n Arguments: {}".format( - tool["name"], - tool["hash"], - tool["description"], - json.dumps(tool["input_schema"]["properties"]), + if tool["hash"] in seen_tool_hashes: + click.echo("- **{}**: `{}`".format(tool["name"], tool["hash"])) + else: + seen_tool_hashes.add(tool["hash"]) + click.echo( + "- **{}**: `{}`
\n{}
\n Arguments: `{}`".format( + tool["name"], + tool["hash"], + textwrap.indent( + (tool["description"] or "").rstrip(), " " + ), + json.dumps(tool["input_schema"]["properties"]), + ) ) - ) if row["tool_results"]: click.echo("\n### Tool results\n") for tool_result in row["tool_results"]: @@ -2211,7 +2258,7 @@ def _display_fragments(fragments, title): "- **{}**: `{}`
\n{}{}{}".format( tool_result["name"], tool_result["tool_call_id"], - textwrap.indent(tool_result["output"], " "), + _fenced_block(tool_result["output"]), ( "
\n **Error**: {}\n".format( tool_result["exception"] @@ -2261,17 +2308,17 @@ def _display_fragments(fragments, title): click.echo("### Tool calls\n") for tool_call in row["tool_calls"]: click.echo( - "- **{}**: `{}`
\n Arguments: {}".format( + "- **{}**: `{}`
\n{}".format( tool_call["name"], tool_call["tool_call_id"], - json.dumps(tool_call["arguments"]), + _format_tool_call_arguments(tool_call["arguments"]), ) ) click.echo("") if response: click.echo("{}\n".format(response)) if usage: - token_usage = token_usage_string( + token_usage = _token_usage_markdown( row["input_tokens"], row["output_tokens"], json.loads(row["token_details"]) if row["token_details"] else None, diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index 78b160b08..ffdf3c7b0 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -171,6 +171,41 @@ def test_logs_text_with_options(user_path): assert "- media_resolution: low" in output +def test_logs_token_usage_details_are_markdown_code(user_path): + log_path = str(user_path / "logs_token_details.db") + db = sqlite_utils.Database(log_path) + migrate(db) + db["responses"].insert( + { + "id": str(monotonic_ulid()).lower(), + "system": None, + "prompt": "prompt", + "response": "response", + "model": "davinci", + "datetime_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "conversation_id": "abc123", + "input_tokens": 2, + "output_tokens": 5, + "token_details": json.dumps( + { + "output_tokens_details": { + "reasoning_tokens": 1, + "label": "`reasoning`", + } + } + ), + } + ) + + runner = CliRunner() + result = runner.invoke(cli, ["logs", "-p", log_path, "-u"], catch_exceptions=False) + assert result.exit_code == 0 + assert ( + '## Token usage\n\n2 input, 5 output, ``{"output_tokens_details": ' + '{"reasoning_tokens": 1, "label": "`reasoning`"}}``\n' + ) in result.output + + @pytest.mark.parametrize("n", (None, 0, 2)) def test_logs_json(n, log_path): "Test that logs command correctly returns requested -n records" @@ -955,9 +990,11 @@ def demo(): "### Tool results\n" "\n" "- **demo**: `tc_TCID`
\n" + " ```\n" " one\n" " two\n" " three\n" + " ```\n" "\n" ) in normalized_output # Log one that did NOT use tools, check that `llm logs --tools` ignores it @@ -968,6 +1005,46 @@ def demo(): assert "three" in logs_tools_output +def test_logs_tool_call_argument_formatting(logs_db): + runner = CliRunner() + code = textwrap.dedent(""" + def demo(timeout: int, options: list): + return "ok" + """) + result1 = runner.invoke( + cli, + [ + "-m", + "echo", + "--functions", + code, + json.dumps( + { + "tool_calls": [ + { + "name": "demo", + "arguments": { + "timeout": 120, + "options": ["`tick`"], + }, + } + ] + } + ), + ], + ) + assert result1.exit_code == 0 + result2 = runner.invoke(cli, ["logs", "-c"]) + normalized_output = re.sub(r"tc_[0-9a-z]{26}", "tc_TCID", result2.output) + assert ( + "### Tool calls\n" + "\n" + "- **demo**: `tc_TCID`
\n" + " timeout: `120`\n" + ' options: ``["`tick`"]``\n' + ) in normalized_output + + def test_logs_backup(logs_db): assert not logs_db.tables runner = CliRunner() diff --git a/tests/test_tools.py b/tests/test_tools.py index 5f5388ca9..81d01dc4b 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -524,7 +524,9 @@ def test_tool_errors(async_): normalized_log_text = re.sub(r"tc_[0-9a-z]{26}", "tc_TCID", log_text_result.output) assert ( "- **trigger_error**: `tc_TCID`
\n" - " Error: Error!
\n" + " ```\n" + " Error: Error!\n" + " ```
\n" " **Error**: Exception: Error!\n" ) in normalized_log_text From 2462813aa0242fa1a1d332623f3e6e436312ab0c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 10:05:45 -0700 Subject: [PATCH 108/258] Show just first 7 characters of tool hashes Refs https://github.com/simonw/llm/issues/1515#issuecomment-4886840756 --- llm/cli.py | 4 +++- tests/test_llm_logs.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/llm/cli.py b/llm/cli.py index e91f24bab..19e462cb4 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -2226,7 +2226,9 @@ def _display_fragments(fragments, title): click.echo("\n### Tools\n") for tool in row["tools"]: if tool["hash"] in seen_tool_hashes: - click.echo("- **{}**: `{}`".format(tool["name"], tool["hash"])) + click.echo( + "- **{}**: `{}`".format(tool["name"], tool["hash"][:7]) + ) else: seen_tool_hashes.add(tool["hash"]) click.echo( diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index ffdf3c7b0..566d5a7ea 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -1005,6 +1005,32 @@ def demo(): assert "three" in logs_tools_output +def test_logs_repeated_tools_use_short_hash(logs_db): + runner = CliRunner() + code = textwrap.dedent(""" + def demo(): + return "ok" + """) + args = [ + "-m", + "echo", + "--functions", + code, + json.dumps({"tool_calls": [{"name": "demo"}]}), + ] + result1 = runner.invoke(cli, args) + assert result1.exit_code == 0 + result2 = runner.invoke(cli, args) + assert result2.exit_code == 0 + + result3 = runner.invoke(cli, ["logs", "-n", "2"]) + assert result3.exit_code == 0 + tool_hashes = re.findall(r"- \*\*demo\*\*: `([0-9a-f]+)`", result3.output) + assert len(tool_hashes) == 2 + assert len(tool_hashes[0]) == 64 + assert tool_hashes[1] == tool_hashes[0][:7] + + def test_logs_tool_call_argument_formatting(logs_db): runner = CliRunner() code = textwrap.dedent(""" From 67adad2c10be5c1898e3e1a664adb573f5d032cf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 10:17:34 -0700 Subject: [PATCH 109/258] Use db.execute() not db.query() This gets us working with sqlite-utils 4.0rc2 Refs https://github.com/simonw/sqlite-utils/issues/758 --- llm/embeddings_migrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm/embeddings_migrations.py b/llm/embeddings_migrations.py index 69545f3ea..96444bd65 100644 --- a/llm/embeddings_migrations.py +++ b/llm/embeddings_migrations.py @@ -32,7 +32,7 @@ def m003_add_updated(db): # Pretty-print the schema db["embeddings"].transform() # Assume anything existing was last updated right now - db.query( + db.execute( "update embeddings set updated = ? where updated is null", [int(time.time())] ) From ba46960fe9e36dce612833f5bbd66aea61e7bdde Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 10:19:23 -0700 Subject: [PATCH 110/258] Test against sqlite-utils 4.0rc2 Refs https://github.com/simonw/sqlite-utils/issues/758#issuecomment-4886879441 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ef5d0d0f2..0a80b33ce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: include: - os: ubuntu-latest python-version: "3.14" - sqlite-utils-version: "4.0rc1" + sqlite-utils-version: "4.0rc2" steps: - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} From ef2eadf768602d20447fc7b7a392d7bd1cd01c1f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 10:32:16 -0700 Subject: [PATCH 111/258] Only run cog check on sqlite-utils 4.0rc2 Refs https://github.com/simonw/sqlite-utils/issues/758#issuecomment-4886940409 --- .github/workflows/test.yml | 2 +- Justfile | 4 +- docs/embeddings/python-api.md | 26 ++--- docs/logging.md | 186 +++++++++++++++++----------------- 4 files changed, 109 insertions(+), 109 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0a80b33ce..9644715be 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,7 +36,7 @@ jobs: run: | python -m pytest -vv - name: Check if cog needs to be run - if: matrix.os != 'windows-latest' + if: matrix.sqlite-utils-version == '4.0rc2' run: | cog --check \ -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" \ diff --git a/Justfile b/Justfile index 626ff09c4..c74742bca 100644 --- a/Justfile +++ b/Justfile @@ -11,7 +11,7 @@ echo " Black" uv run black . --check echo " cog" - uv run cog --check \ + uv run --with sqlite-utils==4.0rc2 cog --check \ -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" \ README.md docs/*.md echo " mypy" @@ -25,7 +25,7 @@ # Rebuild docs with cog @cog: - uv run cog -r -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" docs/**/*.md docs/*.md README.md + uv run --with sqlite-utils==4.0rc2 cog -r -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" docs/**/*.md docs/*.md README.md # Serve live docs on localhost:8000 @docs: cog diff --git a/docs/embeddings/python-api.md b/docs/embeddings/python-api.md index ca586991c..c6ec44e42 100644 --- a/docs/embeddings/python-api.md +++ b/docs/embeddings/python-api.md @@ -191,21 +191,21 @@ for table in ("collections", "embeddings"): cog.out("```\n") ]]] --> ```sql -CREATE TABLE [collections] ( - [id] INTEGER PRIMARY KEY, - [name] TEXT, - [model] TEXT +CREATE TABLE "collections" ( + "id" INTEGER PRIMARY KEY, + "name" TEXT, + "model" TEXT ) CREATE TABLE "embeddings" ( - [collection_id] INTEGER REFERENCES [collections]([id]), - [id] TEXT, - [embedding] BLOB, - [content] TEXT, - [content_blob] BLOB, - [content_hash] BLOB, - [metadata] TEXT, - [updated] INTEGER, - PRIMARY KEY ([collection_id], [id]) + "collection_id" INTEGER REFERENCES "collections"("id"), + "id" TEXT, + "embedding" BLOB, + "content" TEXT, + "content_blob" BLOB, + "content_hash" BLOB, + "metadata" TEXT, + "updated" INTEGER, + PRIMARY KEY ("collection_id", "id") ) ``` diff --git a/docs/logging.md b/docs/logging.md index 75d6be0f9..4fba39f75 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -308,117 +308,117 @@ for table in ( cog.out("```\n") ]]] --> ```sql -CREATE TABLE [conversations] ( - [id] TEXT PRIMARY KEY, - [name] TEXT, - [model] TEXT +CREATE TABLE "conversations" ( + "id" TEXT PRIMARY KEY, + "name" TEXT, + "model" TEXT ); -CREATE TABLE [schemas] ( - [id] TEXT PRIMARY KEY, - [content] TEXT +CREATE TABLE "schemas" ( + "id" TEXT PRIMARY KEY, + "content" TEXT ); CREATE TABLE "responses" ( - [id] TEXT PRIMARY KEY, - [model] TEXT, - [prompt] TEXT, - [system] TEXT, - [prompt_json] TEXT, - [options_json] TEXT, - [response] TEXT, - [response_json] TEXT, - [conversation_id] TEXT REFERENCES [conversations]([id]), - [duration_ms] INTEGER, - [datetime_utc] TEXT, - [input_tokens] INTEGER, - [output_tokens] INTEGER, - [token_details] TEXT, - [schema_id] TEXT REFERENCES [schemas]([id]), - [resolved_model] TEXT, - [reasoning] TEXT + "id" TEXT PRIMARY KEY, + "model" TEXT, + "prompt" TEXT, + "system" TEXT, + "prompt_json" TEXT, + "options_json" TEXT, + "response" TEXT, + "response_json" TEXT, + "conversation_id" TEXT REFERENCES "conversations"("id"), + "duration_ms" INTEGER, + "datetime_utc" TEXT, + "input_tokens" INTEGER, + "output_tokens" INTEGER, + "token_details" TEXT, + "schema_id" TEXT REFERENCES "schemas"("id"), + "resolved_model" TEXT, + "reasoning" TEXT ); -CREATE VIRTUAL TABLE [responses_fts] USING FTS5 ( - [prompt], - [response], - content=[responses] +CREATE VIRTUAL TABLE "responses_fts" USING FTS5 ( + "prompt", + "response", + content="responses" ); -CREATE TABLE [attachments] ( - [id] TEXT PRIMARY KEY, - [type] TEXT, - [path] TEXT, - [url] TEXT, - [content] BLOB +CREATE TABLE "attachments" ( + "id" TEXT PRIMARY KEY, + "type" TEXT, + "path" TEXT, + "url" TEXT, + "content" BLOB ); -CREATE TABLE [prompt_attachments] ( - [response_id] TEXT REFERENCES [responses]([id]), - [attachment_id] TEXT REFERENCES [attachments]([id]), - [order] INTEGER, - PRIMARY KEY ([response_id], - [attachment_id]) +CREATE TABLE "prompt_attachments" ( + "response_id" TEXT REFERENCES "responses"("id"), + "attachment_id" TEXT REFERENCES "attachments"("id"), + "order" INTEGER, + PRIMARY KEY ("response_id", + "attachment_id") ); -CREATE TABLE [fragments] ( - [id] INTEGER PRIMARY KEY, - [hash] TEXT, - [content] TEXT, - [datetime_utc] TEXT, - [source] TEXT +CREATE TABLE "fragments" ( + "id" INTEGER PRIMARY KEY, + "hash" TEXT, + "content" TEXT, + "datetime_utc" TEXT, + "source" TEXT ); -CREATE TABLE [fragment_aliases] ( - [alias] TEXT PRIMARY KEY, - [fragment_id] INTEGER REFERENCES [fragments]([id]) +CREATE TABLE "fragment_aliases" ( + "alias" TEXT PRIMARY KEY, + "fragment_id" INTEGER REFERENCES "fragments"("id") ); CREATE TABLE "prompt_fragments" ( - [response_id] TEXT REFERENCES [responses]([id]), - [fragment_id] INTEGER REFERENCES [fragments]([id]), - [order] INTEGER, - PRIMARY KEY ([response_id], - [fragment_id], - [order]) + "response_id" TEXT REFERENCES "responses"("id"), + "fragment_id" INTEGER REFERENCES "fragments"("id"), + "order" INTEGER, + PRIMARY KEY ("response_id", + "fragment_id", + "order") ); CREATE TABLE "system_fragments" ( - [response_id] TEXT REFERENCES [responses]([id]), - [fragment_id] INTEGER REFERENCES [fragments]([id]), - [order] INTEGER, - PRIMARY KEY ([response_id], - [fragment_id], - [order]) + "response_id" TEXT REFERENCES "responses"("id"), + "fragment_id" INTEGER REFERENCES "fragments"("id"), + "order" INTEGER, + PRIMARY KEY ("response_id", + "fragment_id", + "order") ); -CREATE TABLE [tools] ( - [id] INTEGER PRIMARY KEY, - [hash] TEXT, - [name] TEXT, - [description] TEXT, - [input_schema] TEXT, - [plugin] TEXT +CREATE TABLE "tools" ( + "id" INTEGER PRIMARY KEY, + "hash" TEXT, + "name" TEXT, + "description" TEXT, + "input_schema" TEXT, + "plugin" TEXT ); -CREATE TABLE [tool_responses] ( - [tool_id] INTEGER REFERENCES [tools]([id]), - [response_id] TEXT REFERENCES [responses]([id]), - PRIMARY KEY ([tool_id], - [response_id]) +CREATE TABLE "tool_responses" ( + "tool_id" INTEGER REFERENCES "tools"("id"), + "response_id" TEXT REFERENCES "responses"("id"), + PRIMARY KEY ("tool_id", + "response_id") ); -CREATE TABLE [tool_calls] ( - [id] INTEGER PRIMARY KEY, - [response_id] TEXT REFERENCES [responses]([id]), - [tool_id] INTEGER REFERENCES [tools]([id]), - [name] TEXT, - [arguments] TEXT, - [tool_call_id] TEXT +CREATE TABLE "tool_calls" ( + "id" INTEGER PRIMARY KEY, + "response_id" TEXT REFERENCES "responses"("id"), + "tool_id" INTEGER REFERENCES "tools"("id"), + "name" TEXT, + "arguments" TEXT, + "tool_call_id" TEXT ); CREATE TABLE "tool_results" ( - [id] INTEGER PRIMARY KEY, - [response_id] TEXT REFERENCES [responses]([id]), - [tool_id] INTEGER REFERENCES [tools]([id]), - [name] TEXT, - [output] TEXT, - [tool_call_id] TEXT, - [instance_id] INTEGER REFERENCES [tool_instances]([id]), - [exception] TEXT + "id" INTEGER PRIMARY KEY, + "response_id" TEXT REFERENCES "responses"("id"), + "tool_id" INTEGER REFERENCES "tools"("id"), + "name" TEXT, + "output" TEXT, + "tool_call_id" TEXT, + "instance_id" INTEGER REFERENCES "tool_instances"("id"), + "exception" TEXT ); -CREATE TABLE [tool_instances] ( - [id] INTEGER PRIMARY KEY, - [plugin] TEXT, - [name] TEXT, - [arguments] TEXT +CREATE TABLE "tool_instances" ( + "id" INTEGER PRIMARY KEY, + "plugin" TEXT, + "name" TEXT, + "arguments" TEXT ); ``` From dfe12787577945137c4ad7ee6a242a229fe90c8e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 22:47:41 -0700 Subject: [PATCH 112/258] Test against sqlite-utils 4.0rc3 Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4889420844 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9644715be..9a2662cd7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: include: - os: ubuntu-latest python-version: "3.14" - sqlite-utils-version: "4.0rc2" + sqlite-utils-version: "4.0rc3" steps: - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} From 16870b77d28f5dd3b3c4c6e7ee419e74d1372879 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 22:42:01 -0700 Subject: [PATCH 113/258] Test LLM against sqlite-utils>=4.0rc4 Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900497417 --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9a2662cd7..38e0da777 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: include: - os: ubuntu-latest python-version: "3.14" - sqlite-utils-version: "4.0rc3" + sqlite-utils-version: "4.0rc4" steps: - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} @@ -36,7 +36,7 @@ jobs: run: | python -m pytest -vv - name: Check if cog needs to be run - if: matrix.sqlite-utils-version == '4.0rc2' + if: matrix.sqlite-utils-version == '4.0rc4' run: | cog --check \ -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" \ From ef6fc13631620d9631ed571a9a85a5fdc96af148 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 9 Jul 2026 09:02:29 -0700 Subject: [PATCH 114/258] Handle tool calls with empty arguments better, closes #1521 --- llm/default_plugins/openai_models.py | 8 +- .../test_tools_streaming_variant_d.yaml | 145 ++++++++++++++++++ tests/test_tools_streaming.py | 12 ++ 3 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 tests/cassettes/test_tools_streaming/test_tools_streaming_variant_d.yaml diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 724497271..45c3fc092 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1007,7 +1007,7 @@ def execute( llm.ToolCall( tool_call_id=value.id, name=value.function.name, - arguments=json.loads(value.function.arguments), + arguments=json.loads(value.function.arguments or "{}"), ) ) else: @@ -1024,7 +1024,7 @@ def execute( llm.ToolCall( tool_call_id=tool_call.id, name=tool_call.function.name, - arguments=json.loads(tool_call.function.arguments), + arguments=json.loads(tool_call.function.arguments or "{}"), ) ) yield StreamEvent( @@ -1122,7 +1122,7 @@ async def execute( llm.ToolCall( tool_call_id=value.id, name=value.function.name, - arguments=json.loads(value.function.arguments), + arguments=json.loads(value.function.arguments or "{}"), ) ) response.response_json = remove_dict_none_values(combine_chunks(chunks)) @@ -1140,7 +1140,7 @@ async def execute( llm.ToolCall( tool_call_id=tool_call.id, name=tool_call.function.name, - arguments=json.loads(tool_call.function.arguments), + arguments=json.loads(tool_call.function.arguments or "{}"), ) ) yield StreamEvent( diff --git a/tests/cassettes/test_tools_streaming/test_tools_streaming_variant_d.yaml b/tests/cassettes/test_tools_streaming/test_tools_streaming_variant_d.yaml new file mode 100644 index 000000000..01c54923b --- /dev/null +++ b/tests/cassettes/test_tools_streaming/test_tools_streaming_variant_d.yaml @@ -0,0 +1,145 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"What is the current llm version?"}],"model":"muse-spark-1.1","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return + the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '315' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.78.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"gen-1753242299-DdArgsNullVariantD00","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242299-DdArgsNullVariantD00","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"0","type":"function","function":{"name":"llm_version","arguments":null}}]},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242299-DdArgsNullVariantD00","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"tool_calls","native_finish_reason":"tool_calls","logprobs":null}],"usage":{"prompt_tokens":57,"completion_tokens":17,"total_tokens":74,"cost":0.00007159,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}} + + + data: [DONE] + + + ' + headers: + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 23 Jul 2025 14:54:09 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"role":"user","content":"What is the current llm version?"},{"role":"assistant","content":""},{"role":"assistant","tool_calls":[{"type":"function","id":"0","function":{"name":"llm_version","arguments":"{}"}}]},{"role":"tool","tool_call_id":"0","content":"0.fixed-version"}],"model":"muse-spark-1.1","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return + the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '517' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.78.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"The"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + current"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + version"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + of"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + *"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"ll"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"m"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"*"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + is"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + **"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"0"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"fixed-version"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"**."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop","native_finish_reason":"stop","logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"usage":{"prompt_tokens":107,"completion_tokens":15,"total_tokens":122,"cost":0.0001017,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}} + + + data: [DONE] + + + ' + headers: + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 23 Jul 2025 14:54:10 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/tests/test_tools_streaming.py b/tests/test_tools_streaming.py index 2e3967f7c..87a27ac65 100644 --- a/tests/test_tools_streaming.py +++ b/tests/test_tools_streaming.py @@ -37,3 +37,15 @@ def test_tools_streaming_variant_c(): "".join(chain) == "The installed version of LLM on this system is 0.fixed-version." ) + + +# This response contains streaming variant "d" where a no-argument tool call +# streams arguments=null and never sends a "{}" chunk, so the accumulated +# arguments string stays empty - json.loads("") used to raise here. +@pytest.mark.vcr(record_mode="none") +def test_tools_streaming_variant_d(): + model = llm.get_model("gpt-4.1-mini") + chain = model.chain( + "What is the current llm version?", tools=[llm_version], key=API_KEY + ) + assert "".join(chain) == "The current version of *llm* is **0.fixed-version**." From 0392226e6630746ef51ffd309c2bee6a5f72b58e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 9 Jul 2026 09:07:51 -0700 Subject: [PATCH 115/258] Release notes for 0.31.1 Refs #1521 --- docs/changelog.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index b152f27e9..47f617cd0 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,10 @@ # Changelog +(v0_31_1)= +## 0.31.1 (2026-07-09) + +- Fix for a bug with OpenAI Chat Completion endpoints where a tool call with empty arguments could result in a JSON error from some providers. [#1521](https://github.com/simonw/llm/issues/1521) + (v0_32_a3)= ## 0.32a3 (2026-06-09) From 8f81f9222f15dfe579cb113d8191da1ea234700a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 9 Jul 2026 11:51:16 -0700 Subject: [PATCH 116/258] Upgrade to actions/checkout@v7 --- .github/workflows/cog.yml | 2 +- .github/workflows/publish.yml | 4 ++-- .github/workflows/stable-docs.yml | 2 +- .github/workflows/test.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cog.yml b/.github/workflows/cog.yml index 6ff5df5f2..778e0230b 100644 --- a/.github/workflows/cog.yml +++ b/.github/workflows/cog.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: ref: ${{ github.head_ref }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 40e4cee24..e99c0ab6f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: @@ -34,7 +34,7 @@ jobs: id-token: write needs: [test] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml index f55ac85ff..3ae9d9447 100644 --- a/.github/workflows/stable-docs.yml +++ b/.github/workflows/stable-docs.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 # We need all commits to find docs/ changes - name: Set up Git user diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38e0da777..e3ab350af 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,7 @@ jobs: python-version: "3.14" sqlite-utils-version: "4.0rc4" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: From a82cb832198a86d330458017ed3cd1b8b6c85611 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 11:31:38 -0700 Subject: [PATCH 117/258] Documentation for responses: true in extra-openai-models.yaml --- docs/openai-models.md | 2 ++ docs/other-models.md | 1 + 2 files changed, 3 insertions(+) diff --git a/docs/openai-models.md b/docs/openai-models.md index 7cf9d5b32..059f4b3d5 100644 --- a/docs/openai-models.md +++ b/docs/openai-models.md @@ -164,6 +164,8 @@ The `model_id` is the identifier that will be recorded in the LLM logs. You can If the model is a completion model (such as `gpt-3.5-turbo-instruct`) add `completion: true` to the configuration. +If the model should use the OpenAI Responses API rather than Chat Completions, add `responses: true` to the configuration. This is useful for models such as `o1`, `o3-mini` and `gpt-5`-style models that are accessed through `/v1/responses`. + If the model supports structured extraction using json_schema, add `supports_schema: true` to the configuration. For reasoning models like `o1` or `o3-mini` add `reasoning: true`. diff --git a/docs/other-models.md b/docs/other-models.md index 69ed41595..cd65ce38d 100644 --- a/docs/other-models.md +++ b/docs/other-models.md @@ -46,6 +46,7 @@ You can set `api_key_name` to the name of a key stored using the {ref}`api-keys` Other keys you can use here: - `completion: true` for completion models that should use the `/completion` endpoint as opposed to `/completion/chat` +- `responses: true` for models that should use the OpenAI Responses API (`/responses`) instead of the Chat Completions API (`/chat/completions`) - `supports_tools: true` for models that support tool calling - `can_stream: false` to disable streaming mode for models that cannot stream - `supports_schema: true` for models that support JSON structured schema output From 5e9a8b3783b773c3c7f7746f70aedb443411a1a6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 11:31:49 -0700 Subject: [PATCH 118/258] GPT-5.6 model family --- llm/default_plugins/openai_models.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 45c3fc092..724e543de 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -308,6 +308,29 @@ def register_models(register): ), ) + # GPT-5.6 + for model_id in ("gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"): + register( + Responses( + model_id, + vision=True, + reasoning=True, + verbosity=True, + image_detail_original=True, + supports_schema=True, + supports_tools=True, + ), + AsyncResponses( + model_id, + vision=True, + reasoning=True, + verbosity=True, + image_detail_original=True, + supports_schema=True, + supports_tools=True, + ), + ) + # The -instruct completion model register( Completion("gpt-3.5-turbo-instruct", default_max_tokens=256), @@ -576,6 +599,7 @@ class ReasoningEffortEnum(str, Enum): medium = "medium" high = "high" xhigh = "xhigh" + max = "max" class VerbosityEnum(str, Enum): From 1dc49af9a9057eec134cb171ba6f6ec793572275 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 13:24:41 -0700 Subject: [PATCH 119/258] Remove unavailable OpenAI models (#1554) Closes #1553 --- docs/aliases.md | 3 - docs/openai-models.md | 18 +- docs/usage.md | 282 ++++----------------------- llm/default_plugins/openai_models.py | 89 ++------- pyproject.toml | 2 +- ruff.toml | 3 + tests/test_aliases.py | 2 - tests/test_cli_openai_models.py | 115 +++-------- tests/test_llm.py | 4 +- 9 files changed, 93 insertions(+), 425 deletions(-) diff --git a/docs/aliases.md b/docs/aliases.md index ced77a570..33e7e33dd 100644 --- a/docs/aliases.md +++ b/docs/aliases.md @@ -20,7 +20,6 @@ cog.out("```\n{}```".format(result.output)) ]]] --> ``` 4o : gpt-4o -chatgpt-4o : chatgpt-4o-latest 4o-mini : gpt-4o-mini 4.1 : gpt-4.1 4.1-mini : gpt-4.1-mini @@ -31,11 +30,9 @@ chatgpt-16k : gpt-3.5-turbo-16k 3.5-16k : gpt-3.5-turbo-16k 4 : gpt-4 gpt4 : gpt-4 -4-32k : gpt-4-32k gpt-4-turbo-preview : gpt-4-turbo 4-turbo : gpt-4-turbo 4t : gpt-4-turbo -gpt-4.5 : gpt-4.5-preview 3.5-instruct : gpt-3.5-turbo-instruct chatgpt-instruct : gpt-3.5-turbo-instruct ada : text-embedding-ada-002 (embedding) diff --git a/docs/openai-models.md b/docs/openai-models.md index 059f4b3d5..c8c33b79b 100644 --- a/docs/openai-models.md +++ b/docs/openai-models.md @@ -32,30 +32,17 @@ cog.out("```\n{}\n```".format("\n".join(models))) ]]] --> ``` OpenAI Chat: gpt-4o (aliases: 4o) -OpenAI Chat: chatgpt-4o-latest (aliases: chatgpt-4o) OpenAI Chat: gpt-4o-mini (aliases: 4o-mini) -OpenAI Chat: gpt-4o-audio-preview -OpenAI Chat: gpt-4o-audio-preview-2024-12-17 -OpenAI Chat: gpt-4o-audio-preview-2024-10-01 -OpenAI Chat: gpt-4o-mini-audio-preview -OpenAI Chat: gpt-4o-mini-audio-preview-2024-12-17 OpenAI Chat: gpt-4.1 (aliases: 4.1) OpenAI Chat: gpt-4.1-mini (aliases: 4.1-mini) OpenAI Chat: gpt-4.1-nano (aliases: 4.1-nano) OpenAI Chat: gpt-3.5-turbo (aliases: 3.5, chatgpt) OpenAI Chat: gpt-3.5-turbo-16k (aliases: chatgpt-16k, 3.5-16k) OpenAI Chat: gpt-4 (aliases: 4, gpt4) -OpenAI Chat: gpt-4-32k (aliases: 4-32k) -OpenAI Chat: gpt-4-1106-preview -OpenAI Chat: gpt-4-0125-preview OpenAI Chat: gpt-4-turbo-2024-04-09 OpenAI Chat: gpt-4-turbo (aliases: gpt-4-turbo-preview, 4-turbo, 4t) -OpenAI Chat: gpt-4.5-preview-2025-02-27 -OpenAI Chat: gpt-4.5-preview (aliases: gpt-4.5) OpenAI Responses: o1 OpenAI Responses: o1-2024-12-17 -OpenAI Chat: o1-preview -OpenAI Chat: o1-mini OpenAI Responses: o3-mini OpenAI Responses: o3 OpenAI Responses: o4-mini @@ -66,7 +53,6 @@ OpenAI Responses: gpt-5-2025-08-07 OpenAI Responses: gpt-5-mini-2025-08-07 OpenAI Responses: gpt-5-nano-2025-08-07 OpenAI Responses: gpt-5.1 -OpenAI Responses: gpt-5.1-chat-latest OpenAI Responses: gpt-5.2 OpenAI Responses: gpt-5.2-chat-latest OpenAI Responses: gpt-5.4 @@ -77,6 +63,9 @@ OpenAI Responses: gpt-5.4-nano OpenAI Responses: gpt-5.4-nano-2026-03-17 OpenAI Responses: gpt-5.5 OpenAI Responses: gpt-5.5-2026-04-23 +OpenAI Responses: gpt-5.6-sol +OpenAI Responses: gpt-5.6-terra +OpenAI Responses: gpt-5.6-luna OpenAI Completion: gpt-3.5-turbo-instruct (aliases: 3.5-instruct, chatgpt-instruct) ``` @@ -184,7 +173,6 @@ Example output: OpenAI Chat: gpt-3.5-turbo (aliases: 3.5, chatgpt) OpenAI Chat: gpt-3.5-turbo-16k (aliases: chatgpt-16k, 3.5-16k) OpenAI Chat: gpt-4 (aliases: 4, gpt4) -OpenAI Chat: gpt-4-32k (aliases: 4-32k) OpenAI Chat: gpt-3.5-turbo-0613 (aliases: 0613) ``` Running `llm logs -n 1` should confirm that the prompt and response has been correctly logged to the database. diff --git a/docs/usage.md b/docs/usage.md index 1b237f75b..576359a1b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -490,8 +490,8 @@ Example output: ``` OpenAI Chat: gpt-4o (aliases: 4o) OpenAI Chat: gpt-4o-mini (aliases: 4o-mini) -OpenAI Chat: o1-preview -OpenAI Chat: o1-mini +OpenAI Responses: o1 +OpenAI Responses: o3-mini GeminiPro: gemini-1.5-pro-002 GeminiPro: gemini-1.5-flash-002 ... @@ -562,26 +562,6 @@ OpenAI Chat: gpt-4o (aliases: 4o) Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: chatgpt-4o-latest (aliases: chatgpt-4o) - Options: - temperature: float - max_tokens: int - top_p: float - frequency_penalty: float - presence_penalty: float - stop: str - logit_bias: dict, str - seed: int - json_object: boolean - image_detail: str - Attachment types: - application/pdf, image/gif, image/jpeg, image/png, image/webp - Features: - - streaming - - async - Keys: - key: openai - env_var: OPENAI_API_KEY OpenAI Chat: gpt-4o-mini (aliases: 4o-mini) Options: temperature: float @@ -604,106 +584,6 @@ OpenAI Chat: gpt-4o-mini (aliases: 4o-mini) Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-4o-audio-preview - Options: - temperature: float - max_tokens: int - top_p: float - frequency_penalty: float - presence_penalty: float - stop: str - logit_bias: dict, str - seed: int - json_object: boolean - image_detail: str - Attachment types: - audio/mpeg, audio/wav - Features: - - streaming - - async - Keys: - key: openai - env_var: OPENAI_API_KEY -OpenAI Chat: gpt-4o-audio-preview-2024-12-17 - Options: - temperature: float - max_tokens: int - top_p: float - frequency_penalty: float - presence_penalty: float - stop: str - logit_bias: dict, str - seed: int - json_object: boolean - image_detail: str - Attachment types: - audio/mpeg, audio/wav - Features: - - streaming - - async - Keys: - key: openai - env_var: OPENAI_API_KEY -OpenAI Chat: gpt-4o-audio-preview-2024-10-01 - Options: - temperature: float - max_tokens: int - top_p: float - frequency_penalty: float - presence_penalty: float - stop: str - logit_bias: dict, str - seed: int - json_object: boolean - image_detail: str - Attachment types: - audio/mpeg, audio/wav - Features: - - streaming - - async - Keys: - key: openai - env_var: OPENAI_API_KEY -OpenAI Chat: gpt-4o-mini-audio-preview - Options: - temperature: float - max_tokens: int - top_p: float - frequency_penalty: float - presence_penalty: float - stop: str - logit_bias: dict, str - seed: int - json_object: boolean - image_detail: str - Attachment types: - audio/mpeg, audio/wav - Features: - - streaming - - async - Keys: - key: openai - env_var: OPENAI_API_KEY -OpenAI Chat: gpt-4o-mini-audio-preview-2024-12-17 - Options: - temperature: float - max_tokens: int - top_p: float - frequency_penalty: float - presence_penalty: float - stop: str - logit_bias: dict, str - seed: int - json_object: boolean - image_detail: str - Attachment types: - audio/mpeg, audio/wav - Features: - - streaming - - async - Keys: - key: openai - env_var: OPENAI_API_KEY OpenAI Chat: gpt-4.1 (aliases: 4.1) Options: temperature: float @@ -824,60 +704,6 @@ OpenAI Chat: gpt-4 (aliases: 4, gpt4) Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-4-32k (aliases: 4-32k) - Options: - temperature: float - max_tokens: int - top_p: float - frequency_penalty: float - presence_penalty: float - stop: str - logit_bias: dict, str - seed: int - json_object: boolean - image_detail: str - Features: - - streaming - - async - Keys: - key: openai - env_var: OPENAI_API_KEY -OpenAI Chat: gpt-4-1106-preview - Options: - temperature: float - max_tokens: int - top_p: float - frequency_penalty: float - presence_penalty: float - stop: str - logit_bias: dict, str - seed: int - json_object: boolean - image_detail: str - Features: - - streaming - - async - Keys: - key: openai - env_var: OPENAI_API_KEY -OpenAI Chat: gpt-4-0125-preview - Options: - temperature: float - max_tokens: int - top_p: float - frequency_penalty: float - presence_penalty: float - stop: str - logit_bias: dict, str - seed: int - json_object: boolean - image_detail: str - Features: - - streaming - - async - Keys: - key: openai - env_var: OPENAI_API_KEY OpenAI Chat: gpt-4-turbo-2024-04-09 Options: temperature: float @@ -914,50 +740,6 @@ OpenAI Chat: gpt-4-turbo (aliases: gpt-4-turbo-preview, 4-turbo, 4t) Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: gpt-4.5-preview-2025-02-27 - Options: - temperature: float - max_tokens: int - top_p: float - frequency_penalty: float - presence_penalty: float - stop: str - logit_bias: dict, str - seed: int - json_object: boolean - image_detail: str - Attachment types: - application/pdf, image/gif, image/jpeg, image/png, image/webp - Features: - - streaming - - schemas - - tools - - async - Keys: - key: openai - env_var: OPENAI_API_KEY -OpenAI Chat: gpt-4.5-preview (aliases: gpt-4.5) - Options: - temperature: float - max_tokens: int - top_p: float - frequency_penalty: float - presence_penalty: float - stop: str - logit_bias: dict, str - seed: int - json_object: boolean - image_detail: str - Attachment types: - application/pdf, image/gif, image/jpeg, image/png, image/webp - Features: - - streaming - - schemas - - tools - - async - Keys: - key: openai - env_var: OPENAI_API_KEY OpenAI Responses: o1 Options: temperature: float @@ -1033,7 +815,7 @@ OpenAI Responses: o1-2024-12-17 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: o1-preview +OpenAI Responses: o3-mini Options: temperature: float max_tokens: int @@ -1044,14 +826,18 @@ OpenAI Chat: o1-preview logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str + reasoning_effort: str Features: - streaming + - schemas + - tools - async Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Chat: o1-mini +OpenAI Responses: o3 Options: temperature: float max_tokens: int @@ -1062,14 +848,20 @@ OpenAI Chat: o1-mini logit_bias: dict, str seed: int json_object: boolean + chat_completions: boolean image_detail: str + reasoning_effort: str + Attachment types: + application/pdf, image/gif, image/jpeg, image/png, image/webp Features: - streaming + - schemas + - tools - async Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: o3-mini +OpenAI Responses: o4-mini Options: temperature: float max_tokens: int @@ -1083,6 +875,8 @@ OpenAI Responses: o3-mini chat_completions: boolean image_detail: str reasoning_effort: str + Attachment types: + application/pdf, image/gif, image/jpeg, image/png, image/webp Features: - streaming - schemas @@ -1091,7 +885,7 @@ OpenAI Responses: o3-mini Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: o3 +OpenAI Responses: gpt-5 Options: temperature: float max_tokens: int @@ -1105,6 +899,7 @@ OpenAI Responses: o3 chat_completions: boolean image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1115,7 +910,7 @@ OpenAI Responses: o3 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: o4-mini +OpenAI Responses: gpt-5-mini Options: temperature: float max_tokens: int @@ -1129,6 +924,7 @@ OpenAI Responses: o4-mini chat_completions: boolean image_detail: str reasoning_effort: str + verbosity: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1139,7 +935,7 @@ OpenAI Responses: o4-mini Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5 +OpenAI Responses: gpt-5-nano Options: temperature: float max_tokens: int @@ -1164,7 +960,7 @@ OpenAI Responses: gpt-5 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5-mini +OpenAI Responses: gpt-5-2025-08-07 Options: temperature: float max_tokens: int @@ -1189,7 +985,7 @@ OpenAI Responses: gpt-5-mini Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5-nano +OpenAI Responses: gpt-5-mini-2025-08-07 Options: temperature: float max_tokens: int @@ -1214,7 +1010,7 @@ OpenAI Responses: gpt-5-nano Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5-2025-08-07 +OpenAI Responses: gpt-5-nano-2025-08-07 Options: temperature: float max_tokens: int @@ -1239,7 +1035,7 @@ OpenAI Responses: gpt-5-2025-08-07 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5-mini-2025-08-07 +OpenAI Responses: gpt-5.1 Options: temperature: float max_tokens: int @@ -1264,7 +1060,7 @@ OpenAI Responses: gpt-5-mini-2025-08-07 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5-nano-2025-08-07 +OpenAI Responses: gpt-5.2 Options: temperature: float max_tokens: int @@ -1289,7 +1085,7 @@ OpenAI Responses: gpt-5-nano-2025-08-07 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5.1 +OpenAI Responses: gpt-5.2-chat-latest Options: temperature: float max_tokens: int @@ -1314,7 +1110,7 @@ OpenAI Responses: gpt-5.1 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5.1-chat-latest +OpenAI Responses: gpt-5.4 Options: temperature: float max_tokens: int @@ -1339,7 +1135,7 @@ OpenAI Responses: gpt-5.1-chat-latest Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5.2 +OpenAI Responses: gpt-5.4-2026-03-05 Options: temperature: float max_tokens: int @@ -1364,7 +1160,7 @@ OpenAI Responses: gpt-5.2 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5.2-chat-latest +OpenAI Responses: gpt-5.4-mini Options: temperature: float max_tokens: int @@ -1389,7 +1185,7 @@ OpenAI Responses: gpt-5.2-chat-latest Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5.4 +OpenAI Responses: gpt-5.4-mini-2026-03-17 Options: temperature: float max_tokens: int @@ -1414,7 +1210,7 @@ OpenAI Responses: gpt-5.4 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5.4-2026-03-05 +OpenAI Responses: gpt-5.4-nano Options: temperature: float max_tokens: int @@ -1439,7 +1235,7 @@ OpenAI Responses: gpt-5.4-2026-03-05 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5.4-mini +OpenAI Responses: gpt-5.4-nano-2026-03-17 Options: temperature: float max_tokens: int @@ -1464,7 +1260,7 @@ OpenAI Responses: gpt-5.4-mini Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5.4-mini-2026-03-17 +OpenAI Responses: gpt-5.5 Options: temperature: float max_tokens: int @@ -1489,7 +1285,7 @@ OpenAI Responses: gpt-5.4-mini-2026-03-17 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5.4-nano +OpenAI Responses: gpt-5.5-2026-04-23 Options: temperature: float max_tokens: int @@ -1514,7 +1310,7 @@ OpenAI Responses: gpt-5.4-nano Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5.4-nano-2026-03-17 +OpenAI Responses: gpt-5.6-sol Options: temperature: float max_tokens: int @@ -1539,7 +1335,7 @@ OpenAI Responses: gpt-5.4-nano-2026-03-17 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5.5 +OpenAI Responses: gpt-5.6-terra Options: temperature: float max_tokens: int @@ -1564,7 +1360,7 @@ OpenAI Responses: gpt-5.5 Keys: key: openai env_var: OPENAI_API_KEY -OpenAI Responses: gpt-5.5-2026-04-23 +OpenAI Responses: gpt-5.6-luna Options: temperature: float max_tokens: int diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 724e543de..79f723f18 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -49,11 +49,6 @@ def register_models(register): AsyncChat("gpt-4o", vision=True, supports_schema=True, supports_tools=True), aliases=("4o",), ) - register( - Chat("chatgpt-4o-latest", vision=True), - AsyncChat("chatgpt-4o-latest", vision=True), - aliases=("chatgpt-4o",), - ) register( Chat("gpt-4o-mini", vision=True, supports_schema=True, supports_tools=True), AsyncChat( @@ -61,17 +56,6 @@ def register_models(register): ), aliases=("4o-mini",), ) - for audio_model_id in ( - "gpt-4o-audio-preview", - "gpt-4o-audio-preview-2024-12-17", - "gpt-4o-audio-preview-2024-10-01", - "gpt-4o-mini-audio-preview", - "gpt-4o-mini-audio-preview-2024-12-17", - ): - register( - Chat(audio_model_id, audio=True), - AsyncChat(audio_model_id, audio=True), - ) # GPT-4.1 for model_id in ("gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano"): register( @@ -89,38 +73,13 @@ def register_models(register): aliases=("chatgpt-16k", "3.5-16k"), ) register(Chat("gpt-4"), AsyncChat("gpt-4"), aliases=("4", "gpt4")) - register(Chat("gpt-4-32k"), AsyncChat("gpt-4-32k"), aliases=("4-32k",)) # GPT-4 Turbo models - register(Chat("gpt-4-1106-preview"), AsyncChat("gpt-4-1106-preview")) - register(Chat("gpt-4-0125-preview"), AsyncChat("gpt-4-0125-preview")) register(Chat("gpt-4-turbo-2024-04-09"), AsyncChat("gpt-4-turbo-2024-04-09")) register( Chat("gpt-4-turbo"), AsyncChat("gpt-4-turbo"), aliases=("gpt-4-turbo-preview", "4-turbo", "4t"), ) - # GPT-4.5 - register( - Chat( - "gpt-4.5-preview-2025-02-27", - vision=True, - supports_schema=True, - supports_tools=True, - ), - AsyncChat( - "gpt-4.5-preview-2025-02-27", - vision=True, - supports_schema=True, - supports_tools=True, - ), - ) - register( - Chat("gpt-4.5-preview", vision=True, supports_schema=True, supports_tools=True), - AsyncChat( - "gpt-4.5-preview", vision=True, supports_schema=True, supports_tools=True - ), - aliases=("gpt-4.5",), - ) # o1 for model_id in ("o1", "o1-2024-12-17"): register( @@ -142,14 +101,6 @@ def register_models(register): ), ) - register( - Chat("o1-preview", allows_system_prompt=False), - AsyncChat("o1-preview", allows_system_prompt=False), - ) - register( - Chat("o1-mini", allows_system_prompt=False), - AsyncChat("o1-mini", allows_system_prompt=False), - ) register( Responses("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), AsyncResponses( @@ -208,28 +159,24 @@ def register_models(register): ), ) # GPT-5.1 - for model_id in ( - "gpt-5.1", - "gpt-5.1-chat-latest", - ): - register( - Responses( - model_id, - vision=True, - reasoning=True, - verbosity=True, - supports_schema=True, - supports_tools=True, - ), - AsyncResponses( - model_id, - vision=True, - reasoning=True, - verbosity=True, - supports_schema=True, - supports_tools=True, - ), - ) + register( + Responses( + "gpt-5.1", + vision=True, + reasoning=True, + verbosity=True, + supports_schema=True, + supports_tools=True, + ), + AsyncResponses( + "gpt-5.1", + vision=True, + reasoning=True, + verbosity=True, + supports_schema=True, + supports_tools=True, + ), + ) # GPT-5.2 for model_id in ("gpt-5.2", "gpt-5.2-chat-latest"): register( diff --git a/pyproject.toml b/pyproject.toml index 55b7b39db..070773e62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "openai>=2.32.0", "click-default-group>=1.2.3", "sqlite-utils>=3.37", - "sqlite-migrate>=0.1a2", + "sqlite-migrate==0.1a2", "pydantic>=2.0.0", "PyYAML", "pluggy", diff --git a/ruff.toml b/ruff.toml index 7a3f6a3f8..1ecd5d03a 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1 +1,4 @@ line-length = 160 + +[lint] +select = ["E4", "E7", "E9", "F"] diff --git a/tests/test_aliases.py b/tests/test_aliases.py index 661eb1583..2949a8007 100644 --- a/tests/test_aliases.py +++ b/tests/test_aliases.py @@ -37,7 +37,6 @@ def test_cli_aliases_list(args): "3.5-16k : gpt-3.5-turbo-16k\n" "4 : gpt-4\n" "gpt4 : gpt-4\n" - "4-32k : gpt-4-32k\n" "e-demo : embed-demo (embedding)\n" "ada : text-embedding-ada-002 (embedding)\n" ).split("\n"): @@ -64,7 +63,6 @@ def test_cli_aliases_list_json(args): "3.5-16k": "gpt-3.5-turbo-16k", "4": "gpt-4", "gpt4": "gpt-4", - "4-32k": "gpt-4-32k", "ada": "text-embedding-ada-002", "e-demo": "embed-demo", }.items() diff --git a/tests/test_cli_openai_models.py b/tests/test_cli_openai_models.py index 85d753120..7da3fb442 100644 --- a/tests/test_cli_openai_models.py +++ b/tests/test_cli_openai_models.py @@ -77,12 +77,38 @@ def test_gpt5_models_support_verbosity_option(model_id): assert "verbosity" in llm.get_async_model(model_id).Options.model_fields -@pytest.mark.parametrize("model_id", ("gpt-4o", "gpt-4.5-preview", "o3", "o4-mini")) +@pytest.mark.parametrize("model_id", ("gpt-4o", "o3", "o4-mini")) def test_non_gpt5_openai_chat_models_do_not_support_verbosity_option(model_id): assert "verbosity" not in llm.get_model(model_id).Options.model_fields assert "verbosity" not in llm.get_async_model(model_id).Options.model_fields +@pytest.mark.parametrize( + "model_id", + ( + "chatgpt-4o-latest", + "gpt-4o-audio-preview", + "gpt-4o-audio-preview-2024-12-17", + "gpt-4o-audio-preview-2024-10-01", + "gpt-4o-mini-audio-preview", + "gpt-4o-mini-audio-preview-2024-12-17", + "gpt-4-32k", + "gpt-4-1106-preview", + "gpt-4-0125-preview", + "gpt-4.5-preview-2025-02-27", + "gpt-4.5-preview", + "o1-preview", + "o1-mini", + "gpt-5.1-chat-latest", + ), +) +def test_deprecated_models_are_not_registered(model_id): + with pytest.raises(llm.UnknownModelError): + llm.get_model(model_id) + with pytest.raises(llm.UnknownModelError): + llm.get_async_model(model_id) + + def test_gpt5_verbosity_option_is_sent_to_openai_chat_completions(httpx_mock): httpx_mock.add_response( method="POST", @@ -364,93 +390,6 @@ def test_openai_image_detail_original_is_rejected_for_other_models(): assert "Input should be 'low', 'high' or 'auto'" in result.output -@pytest.mark.parametrize("model", ("gpt-4o-mini", "gpt-4o-audio-preview")) -@pytest.mark.parametrize("filetype", ("mp3", "wav")) -def test_only_gpt4_audio_preview_allows_mp3_or_wav(httpx_mock, model, filetype): - httpx_mock.add_response( - method="HEAD", - url=f"https://www.example.com/example.{filetype}", - content=b"binary-data", - headers={"Content-Type": "audio/mpeg" if filetype == "mp3" else "audio/wav"}, - ) - if model == "gpt-4o-audio-preview": - httpx_mock.add_response( - method="POST", - # chat completion request - url="https://api.openai.com/v1/chat/completions", - json={ - "id": "chatcmpl-AQT9a30kxEaM1bqxRPepQsPlCyGJh", - "object": "chat.completion", - "created": 1730871958, - "model": "gpt-4o-audio-preview-2024-10-01", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Why did the pelican get kicked out of the restaurant?\n\nBecause he had a big bill and no way to pay it!", - "refusal": None, - }, - "finish_reason": "stop", - } - ], - "usage": { - "prompt_tokens": 55, - "completion_tokens": 25, - "total_tokens": 80, - "prompt_tokens_details": { - "cached_tokens": 0, - "audio_tokens": 44, - "text_tokens": 11, - "image_tokens": 0, - }, - "completion_tokens_details": { - "reasoning_tokens": 0, - "audio_tokens": 0, - "text_tokens": 25, - "accepted_prediction_tokens": 0, - "rejected_prediction_tokens": 0, - }, - }, - "system_fingerprint": "fp_49254d0e9b", - }, - headers={"Content-Type": "application/json"}, - ) - httpx_mock.add_response( - method="GET", - url=f"https://www.example.com/example.{filetype}", - content=b"binary-data", - headers={ - "Content-Type": "audio/mpeg" if filetype == "mp3" else "audio/wav" - }, - ) - runner = CliRunner() - result = runner.invoke( - cli, - [ - "-m", - model, - "-a", - f"https://www.example.com/example.{filetype}", - "--no-stream", - "--key", - "x", - ], - ) - if model == "gpt-4o-audio-preview": - assert result.exit_code == 0 - assert result.output == ( - "Why did the pelican get kicked out of the restaurant?\n\n" - "Because he had a big bill and no way to pay it!\n" - ) - else: - assert result.exit_code == 1 - long = "audio/mpeg" if filetype == "mp3" else "audio/wav" - assert ( - f"This model does not support attachments of type '{long}'" in result.output - ) - - @pytest.mark.parametrize("async_", (False, True)) @pytest.mark.parametrize("usage", (None, "-u", "--usage")) def test_gpt4o_mini_sync_and_async(monkeypatch, tmpdir, httpx_mock, async_, usage): diff --git a/tests/test_llm.py b/tests/test_llm.py index fc5c1d5d5..29262abeb 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -554,8 +554,8 @@ def test_llm_models_async(user_path): ["OpenAI Chat: gpt-4o "], ), ( - ["-m", "gpt-4o-mini", "-m", "gpt-4.5"], - ["OpenAI Chat: gpt-4o-mini", "OpenAI Chat: gpt-4.5"], + ["-m", "gpt-4o-mini", "-m", "4.1"], + ["OpenAI Chat: gpt-4o-mini", "OpenAI Chat: gpt-4.1"], ["OpenAI Chat: gpt-4o "], ), ), From c12fd50b9f201f6a56cfbc0dd872e4075c35f5ef Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 14:16:47 -0700 Subject: [PATCH 120/258] Ruff >= 0.16.0 (#1557) * ruff>=0.16.0 * Fixed all ruff issues with --fix and --unsafe-fixes * Codex (GPT-5.6 Sol High) fixed remaining Ruff errors: https://gist.github.com/simonw/53404d27979d28f66ae59564d9fb3382 * Ruff target-version = "py310" --- docs/conf.py | 3 - docs/plugins/llm-markov/llm_markov.py | 11 +- llm/__init__.py | 105 ++--- llm/cli.py | 324 ++++++------- llm/default_plugins/openai_models.py | 183 ++++---- llm/embeddings.py | 53 ++- llm/embeddings_migrations.py | 3 +- llm/hookspecs.py | 3 +- llm/migrations.py | 4 +- llm/models.py | 650 +++++++++++++------------- llm/parts.py | 66 ++- llm/plugins.py | 6 +- llm/serialization.py | 42 +- llm/templates.py | 45 +- llm/tools.py | 4 +- llm/utils.py | 63 +-- pyproject.toml | 2 +- ruff.toml | 4 +- tests/conftest.py | 23 +- tests/test_aliases.py | 31 +- tests/test_async.py | 3 +- tests/test_async_parity.py | 5 +- tests/test_attachments.py | 12 +- tests/test_chat.py | 14 +- tests/test_chat_templates.py | 6 +- tests/test_cli_openai_models.py | 8 +- tests/test_cli_options.py | 6 +- tests/test_embed.py | 12 +- tests/test_embed_cli.py | 14 +- tests/test_encode_decode.py | 5 +- tests/test_fragments_cli.py | 16 +- tests/test_keys.py | 10 +- tests/test_llm.py | 14 +- tests/test_llm_logs.py | 21 +- tests/test_migrate.py | 7 +- tests/test_openai_messages.py | 2 +- tests/test_openai_responses.py | 15 +- tests/test_options_parameter.py | 3 +- tests/test_parts.py | 17 +- tests/test_plugins.py | 43 +- tests/test_serialization.py | 7 +- tests/test_templates.py | 16 +- tests/test_tools.py | 18 +- tests/test_tools_streaming.py | 6 +- tests/test_utils.py | 33 +- 45 files changed, 978 insertions(+), 960 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index f4a2d953c..7783599a2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - from subprocess import PIPE, Popen # This file is execfile()d with the current directory set to its diff --git a/docs/plugins/llm-markov/llm_markov.py b/docs/plugins/llm-markov/llm_markov.py index 3ba4d9d8f..4b5d86585 100644 --- a/docs/plugins/llm-markov/llm_markov.py +++ b/docs/plugins/llm-markov/llm_markov.py @@ -1,8 +1,9 @@ -import llm import random import time -from typing import Optional -from pydantic import field_validator, Field + +from pydantic import Field, field_validator + +import llm @llm.hookimpl @@ -35,10 +36,10 @@ class Markov(llm.Model): can_stream = True class Options(llm.Options): - length: Optional[int] = Field( + length: int | None = Field( description="Number of words to generate", default=None ) - delay: Optional[float] = Field( + delay: float | None = Field( description="Seconds to delay between each token", default=None ) diff --git a/llm/__init__.py b/llm/__init__.py index 52d3564ee..1bdfea4fc 100644 --- a/llm/__init__.py +++ b/llm/__init__.py @@ -1,8 +1,19 @@ -from .hookspecs import hookimpl +import inspect +import json +import os +import pathlib +import struct +from collections.abc import Callable +from typing import Any + +import click + +from .embeddings import Collection from .errors import ( ModelError, NeedsKeyException, ) +from .hookspecs import hookimpl from .models import ( AsyncConversation, AsyncKeyModel, @@ -10,7 +21,6 @@ AsyncResponse, Attachment, CancelToolCall, - PauseChain, Conversation, EmbeddingModel, EmbeddingModelWithAliases, @@ -18,6 +28,7 @@ Model, ModelWithAliases, Options, + PauseChain, Prompt, Response, Tool, @@ -34,33 +45,20 @@ tool_message, user, ) -from .utils import schema_dsl, Fragment -from .embeddings import Collection +from .plugins import load_plugins, pm from .templates import Template -from .plugins import pm, load_plugins -import click -from typing import Any, Dict, List, Optional, Callable, Type, Union -import inspect -import json -import os -import pathlib -import struct +from .utils import Fragment, schema_dsl __all__ = [ "AsyncConversation", "AsyncKeyModel", "AsyncModel", "AsyncResponse", - "assistant", "Attachment", "CancelToolCall", "Collection", "Conversation", "Fragment", - "get_async_model", - "get_key", - "get_model", - "hookimpl", "KeyModel", "Message", "Model", @@ -70,16 +68,21 @@ "PauseChain", "Prompt", "Response", - "schema_dsl", - "system", "Template", "Tool", - "Toolbox", "ToolCall", - "tool_message", "ToolOutput", "ToolResult", + "Toolbox", "Usage", + "assistant", + "get_async_model", + "get_key", + "get_model", + "hookimpl", + "schema_dsl", + "system", + "tool_message", "user", "user_dir", ] @@ -106,12 +109,12 @@ def get_plugins(all=False): return plugins -def get_models_with_aliases() -> List["ModelWithAliases"]: +def get_models_with_aliases() -> list["ModelWithAliases"]: model_aliases = [] # Include aliases from aliases.json aliases_path = user_dir() / "aliases.json" - extra_model_aliases: Dict[str, list] = {} + extra_model_aliases: dict[str, list] = {} if aliases_path.exists(): configured_aliases = json.loads(aliases_path.read_text()) for alias, model_id in configured_aliases.items(): @@ -129,7 +132,7 @@ def register(model, async_model=None, aliases=None): return model_aliases -def _get_loaders(hook_method) -> Dict[str, Callable]: +def _get_loaders(hook_method) -> dict[str, Callable]: load_plugins() loaders = {} @@ -145,32 +148,32 @@ def register(prefix, loader): return loaders -def get_template_loaders() -> Dict[str, Callable[[str], Template]]: +def get_template_loaders() -> dict[str, Callable[[str], Template]]: """Get template loaders registered by plugins.""" return _get_loaders(pm.hook.register_template_loaders) -def get_fragment_loaders() -> Dict[ +def get_fragment_loaders() -> dict[ str, - Callable[[str], Union[Fragment, Attachment, List[Union[Fragment, Attachment]]]], + Callable[[str], Fragment | Attachment | list[Fragment | Attachment]], ]: """Get fragment loaders registered by plugins.""" return _get_loaders(pm.hook.register_fragment_loaders) -def get_tools() -> Dict[str, Union[Tool, Type[Toolbox]]]: +def get_tools() -> dict[str, Tool | type[Toolbox]]: """Return all tools (llm.Tool and llm.Toolbox) registered by plugins.""" load_plugins() - tools: Dict[str, Union[Tool, Type[Toolbox]]] = {} + tools: dict[str, Tool | type[Toolbox]] = {} # Variable to track current plugin name current_plugin_name = None def register( - tool_or_function: Union[Tool, Type[Toolbox], Callable[..., Any]], - name: Optional[str] = None, + tool_or_function: Tool | type[Toolbox] | Callable[..., Any], + name: str | None = None, ) -> None: - tool: Union[Tool, Type[Toolbox], None] = None + tool: Tool | type[Toolbox] | None = None # If it's a Toolbox class, set the plugin field on it if inspect.isclass(tool_or_function): @@ -181,9 +184,7 @@ def register( tool.name = name or tool.__name__ else: raise TypeError( - "Toolbox classes must inherit from llm.Toolbox, {} does not.".format( - tool_or_function.__name__ - ) + f"Toolbox classes must inherit from llm.Toolbox, {tool_or_function.__name__} does not." ) # If it's already a Tool instance, use it directly @@ -231,12 +232,12 @@ def register( return tools -def get_embedding_models_with_aliases() -> List["EmbeddingModelWithAliases"]: +def get_embedding_models_with_aliases() -> list["EmbeddingModelWithAliases"]: model_aliases = [] # Include aliases from aliases.json aliases_path = user_dir() / "aliases.json" - extra_model_aliases: Dict[str, list] = {} + extra_model_aliases: dict[str, list] = {} if aliases_path.exists(): configured_aliases = json.loads(aliases_path.read_text()) for alias, model_id in configured_aliases.items(): @@ -273,7 +274,7 @@ def get_embedding_model(name): raise UnknownModelError("Unknown model: " + str(name)) -def get_embedding_model_aliases() -> Dict[str, EmbeddingModel]: +def get_embedding_model_aliases() -> dict[str, EmbeddingModel]: model_aliases = {} for model_with_aliases in get_embedding_models_with_aliases(): for alias in model_with_aliases.aliases: @@ -282,7 +283,7 @@ def get_embedding_model_aliases() -> Dict[str, EmbeddingModel]: return model_aliases -def get_async_model_aliases() -> Dict[str, AsyncModel]: +def get_async_model_aliases() -> dict[str, AsyncModel]: async_model_aliases = {} for model_with_aliases in get_models_with_aliases(): if model_with_aliases.async_model: @@ -294,7 +295,7 @@ def get_async_model_aliases() -> Dict[str, AsyncModel]: return async_model_aliases -def get_model_aliases() -> Dict[str, Model]: +def get_model_aliases() -> dict[str, Model]: model_aliases = {} for model_with_aliases in get_models_with_aliases(): if model_with_aliases.model: @@ -308,19 +309,19 @@ class UnknownModelError(KeyError): pass -def get_models() -> List[Model]: +def get_models() -> list[Model]: "Get all registered models" models_with_aliases = get_models_with_aliases() return [mwa.model for mwa in models_with_aliases if mwa.model] -def get_async_models() -> List[AsyncModel]: +def get_async_models() -> list[AsyncModel]: "Get all registered async models" models_with_aliases = get_models_with_aliases() return [mwa.async_model for mwa in models_with_aliases if mwa.async_model] -def get_async_model(name: Optional[str] = None) -> AsyncModel: +def get_async_model(name: str | None = None) -> AsyncModel: "Get an async model by name or alias" aliases = get_async_model_aliases() name = name or get_default_model() @@ -339,7 +340,7 @@ def get_async_model(name: Optional[str] = None) -> AsyncModel: raise UnknownModelError("Unknown model: " + name) -def get_model(name: Optional[str] = None, _skip_async: bool = False) -> Model: +def get_model(name: str | None = None, _skip_async: bool = False) -> Model: "Get a model by name or alias" aliases = get_model_aliases() name = name or get_default_model() @@ -361,14 +362,14 @@ def get_model(name: Optional[str] = None, _skip_async: bool = False) -> Model: def get_key( - explicit_key: Optional[str] = None, - key_alias: Optional[str] = None, - env_var: Optional[str] = None, + explicit_key: str | None = None, + key_alias: str | None = None, + env_var: str | None = None, *, - alias: Optional[str] = None, - env: Optional[str] = None, - input: Optional[str] = None, -) -> Optional[str]: + alias: str | None = None, + env: str | None = None, + input: str | None = None, +) -> str | None: """ Return an API key based on a hierarchy of potential sources. You should use the keyword arguments, the positional arguments are here purely for backwards-compatibility with older code. @@ -459,7 +460,7 @@ def remove_alias(alias): except json.decoder.JSONDecodeError: raise KeyError("aliases.json file is not valid JSON") if alias not in current: - raise KeyError("No such alias: {}".format(alias)) + raise KeyError(f"No such alias: {alias}") del current[alias] path.write_text(json.dumps(current, indent=4) + "\n") diff --git a/llm/cli.py b/llm/cli.py index 19e462cb4..ecd8121a7 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -1,50 +1,69 @@ import asyncio -import click -from click_default_group import DefaultGroup -from dataclasses import asdict -from importlib.metadata import version +import base64 +import inspect import io import json import os +import pathlib +import re +import readline +import shutil +import sys +import textwrap +import warnings +from collections.abc import Iterable +from dataclasses import asdict +from importlib.metadata import version +from runpy import run_module +from typing import Any, cast + +import click +import httpx +import pydantic +import sqlite_utils +import yaml +from click_default_group import DefaultGroup +from sqlite_utils.utils import Format, rows_from_file + from llm import ( - Attachment, AsyncConversation, AsyncKeyModel, AsyncResponse, + Attachment, CancelToolCall, Collection, Conversation, Fragment, + KeyModel, Response, Template, Tool, Toolbox, UnknownModelError, - KeyModel, encode, get_async_model, - get_default_model, get_default_embedding_model, - get_embedding_models_with_aliases, - get_embedding_model_aliases, + get_default_model, get_embedding_model, - get_plugins, - get_tools, + get_embedding_model_aliases, + get_embedding_models_with_aliases, get_fragment_loaders, - get_template_loaders, get_model, get_model_aliases, get_models_with_aliases, - user_dir, + get_plugins, + get_template_loaders, + get_tools, + remove_alias, set_alias, - set_default_model, set_default_embedding_model, - remove_alias, + set_default_model, + user_dir, ) -from llm.models import _BaseConversation, ChainResponse +from llm.models import ChainResponse, _BaseConversation from .migrations import migrate -from .plugins import pm, load_plugins +from .plugins import load_plugins, pm from .utils import ( ensure_fragment, extract_fenced_code_block, @@ -63,22 +82,6 @@ token_usage_string, truncate_string, ) -import base64 -import httpx -import inspect -import pathlib -import pydantic -import re -import readline -from runpy import run_module -import shutil -import sqlite_utils -from sqlite_utils.utils import rows_from_file, Format -import sys -import textwrap -from typing import cast, Dict, Optional, Iterable, List, Union, Tuple, Type, Any -import warnings -import yaml warnings.simplefilter("ignore", ResourceWarning) @@ -130,12 +133,12 @@ def validate_fragment_alias(ctx, param, value): def resolve_fragments( db: sqlite_utils.Database, fragments: Iterable[str], allow_attachments: bool = False -) -> List[Union[Fragment, Attachment]]: +) -> list[Fragment | Attachment]: """ Resolve fragment strings into a mixed of llm.Fragment() and llm.Attachment() objects. """ - def _load_by_alias(fragment: str) -> Tuple[Optional[str], Optional[str]]: + def _load_by_alias(fragment: str) -> tuple[str | None, str | None]: rows = list( db.query( """ @@ -152,9 +155,9 @@ def _load_by_alias(fragment: str) -> Tuple[Optional[str], Optional[str]]: return None, None # The fragment strings could be URLs or paths or plugin references - resolved: List[Union[Fragment, Attachment]] = [] + resolved: list[Fragment | Attachment] = [] for fragment in fragments: - if fragment.startswith("http://") or fragment.startswith("https://"): + if fragment.startswith(("http://", "https://")): llm_version = version("llm") headers = {"User-Agent": f"llm/{llm_version} (https://llm.datasette.io/)"} client = httpx.Client( @@ -169,7 +172,7 @@ def _load_by_alias(fragment: str) -> Tuple[Optional[str], Optional[str]]: prefix, rest = fragment.split(":", 1) loaders = get_fragment_loaders() if prefix not in loaders: - raise FragmentNotFound("Unknown fragment prefix: {}".format(prefix)) + raise FragmentNotFound(f"Unknown fragment prefix: {prefix}") loader = loaders[prefix] try: result = loader(rest) @@ -179,15 +182,11 @@ def _load_by_alias(fragment: str) -> Tuple[Optional[str], Optional[str]]: isinstance(r, Attachment) for r in result ): raise FragmentNotFound( - "Fragment loader {} returned a disallowed attachment".format( - prefix - ) + f"Fragment loader {prefix} returned a disallowed attachment" ) resolved.extend(result) - except Exception as ex: - raise FragmentNotFound( - "Could not load fragment {}: {}".format(fragment, ex) - ) + except Exception as ex: # noqa: BLE001 + raise FragmentNotFound(f"Could not load fragment {fragment}: {ex}") else: # Try from the DB content, source = _load_by_alias(fragment) @@ -239,8 +238,6 @@ def process_fragments_in_chat( class AttachmentError(Exception): """Exception raised for errors in attachment resolution.""" - pass - def resolve_attachment(value): """ @@ -310,7 +307,7 @@ def resolve_attachment_with_type(value: str, mimetype: str) -> Attachment: return attachment -def attachment_types_callback(ctx, param, values) -> List[Attachment]: +def attachment_types_callback(ctx, param, values) -> list[Attachment]: collected = [] for value, mimetype in values: collected.append(resolve_attachment_with_type(value, mimetype)) @@ -667,7 +664,7 @@ def read_prompt(): try: to_save["model"] = model_aliases[model_id].model_id except KeyError: - raise click.ClickException("'{}' is not a known model".format(model_id)) + raise click.ClickException(f"'{model_id}' is not a known model") prompt = read_prompt() if prompt: to_save["prompt"] = prompt @@ -828,11 +825,11 @@ def read_prompt(): if options: # Validate with pydantic try: - validated_options = dict( - (key, value) + validated_options = { + key: value for key, value in model.Options(**dict(options)) if value is not None - ) + } except pydantic.ValidationError as ex: raise click.ClickException(render_errors(ex.errors())) @@ -914,7 +911,7 @@ async def inner(): response.astream_events(), show_reasoning=not hide_reasoning, ) - print("") + print() else: response = prompt_method( prompt, @@ -949,7 +946,7 @@ async def inner(): response.stream_events(), show_reasoning=not hide_reasoning, ) - print("") + print() else: text = response.text() if extract or extract_last: @@ -975,7 +972,7 @@ async def inner(): # Show token usage to stderr in yellow click.echo( click.style( - "Token usage: {}".format(response_object.token_usage()), + f"Token usage: {response_object.token_usage()}", fg="yellow", bold=True, ), @@ -1154,7 +1151,7 @@ def chat( try: model = get_model(model_id) except KeyError: - raise click.ClickException("'{}' is not a known model".format(model_id)) + raise click.ClickException(f"'{model_id}' is not a known model") if conversation is None: # Start a fresh conversation for this chat @@ -1172,11 +1169,11 @@ def chat( validated_options = get_model_options(model.model_id) if options: try: - validated_options = dict( - (key, value) + validated_options = { + key: value for key, value in model.Options(**dict(options)) if value is not None - ) + } except pydantic.ValidationError as ex: raise click.ClickException(render_errors(ex.errors())) @@ -1217,7 +1214,7 @@ def chat( except FragmentNotFound as ex: raise click.ClickException(str(ex)) - click.echo("Chatting with {}".format(model.model_id)) + click.echo(f"Chatting with {model.model_id}") click.echo("Type 'exit' or 'quit' to exit") click.echo("Type '!multi' to enter multiple lines, then '!end' to finish") click.echo("Type '!edit' to open your default editor and modify the prompt") @@ -1306,14 +1303,14 @@ def chat( show_reasoning=not hide_reasoning, ) response.log_to_db(db) - print("") + print() def load_conversation( - conversation_id: Optional[str], + conversation_id: str | None, async_=False, database=None, -) -> Optional[_BaseConversation]: +) -> _BaseConversation | None: log_path = pathlib.Path(database) if database else logs_db_path() db = sqlite_utils.Database(log_path) migrate(db) @@ -1327,9 +1324,7 @@ def load_conversation( try: row = cast(sqlite_utils.db.Table, db["conversations"]).get(conversation_id) except sqlite_utils.db.NotFoundError: - raise click.ClickException( - "No conversation found with id={}".format(conversation_id) - ) + raise click.ClickException(f"No conversation found with id={conversation_id}") # Inflate that conversation conversation_class = AsyncConversation if async_ else Conversation response_class = AsyncResponse if async_ else Response @@ -1399,7 +1394,7 @@ def keys_get(name): try: click.echo(keys[name]) except KeyError: - raise click.ClickException("No key found with name '{}'".format(name)) + raise click.ClickException(f"No key found with name '{name}'") @keys.command(name="set") @@ -1449,7 +1444,7 @@ def logs_status(): "Show current status of database logging" path = logs_db_path() if not path.exists(): - click.echo("No log database found at {}".format(path)) + click.echo(f"No log database found at {path}") return if logs_on(): click.echo("Logging is ON for all prompts".format()) @@ -1457,12 +1452,10 @@ def logs_status(): click.echo("Logging is OFF".format()) db = sqlite_utils.Database(path) migrate(db) - click.echo("Found log database at {}".format(path)) + click.echo(f"Found log database at {path}") click.echo("Number of conversations logged:\t{}".format(db["conversations"].count)) click.echo("Number of responses logged:\t{}".format(db["responses"].count)) - click.echo( - "Database file size: \t\t{}".format(_human_readable_size(path.stat().st_size)) - ) + click.echo(f"Database file size: \t\t{_human_readable_size(path.stat().st_size)}") @logs.command(name="backup") @@ -1474,11 +1467,9 @@ def backup(path): db = sqlite_utils.Database(logs_path) try: db.execute("vacuum into ?", [str(path)]) - except Exception as ex: + except Exception as ex: # noqa: BLE001 raise click.ClickException(str(ex)) - click.echo( - "Backed up {} to {}".format(_human_readable_size(path.stat().st_size), path) - ) + click.echo(f"Backed up {_human_readable_size(path.stat().st_size)} to {path}") @logs.command(name="on") @@ -1688,7 +1679,7 @@ def logs_list( path = database path = pathlib.Path(path or logs_db_path()) if not path.exists(): - raise click.ClickException("No log database found at {}".format(path)) + raise click.ClickException(f"No log database found at {path}") db = sqlite_utils.Database(path) migrate(db) @@ -1706,7 +1697,7 @@ def logs_list( if flag[1] ] ) - raise click.ClickException("Cannot use --short and {} together".format(invalid)) + raise click.ClickException(f"Cannot use --short and {invalid} together") if response and not current_conversation and not conversation_id: current_conversation = True @@ -1747,7 +1738,7 @@ def logs_list( limit = "" if count is not None and count > 0: - limit = " limit {}".format(count) + limit = f" limit {count}" sql_format = { "limit": limit, @@ -1797,7 +1788,7 @@ def logs_list( ) """ exists_clauses.append(exists_clause) - sql_params["f{}".format(i)] = fragment_hash + sql_params[f"f{i}"] = fragment_hash where_bits.append(" and ".join(exists_clauses)) @@ -1900,16 +1891,14 @@ def logs_list( response = row["response"] or "" try: decoded = json.loads(response) - new_items = [] if ( isinstance(decoded, dict) and (data_key in decoded) and all(isinstance(item, dict) for item in decoded[data_key]) ): - for item in decoded[data_key]: - new_items.append(item) + new_items = list(decoded[data_key]) else: - new_items.append(decoded) + new_items = [decoded] if data_ids: for item in new_items: item[find_unused_key(item, "response_id")] = row["id"] @@ -2058,7 +2047,7 @@ def _fenced_block(value): while "`" * num_backticks in value: num_backticks += 1 fence = "`" * num_backticks - return textwrap.indent("{}\n{}\n{}".format(fence, value, fence), " ") + return textwrap.indent(f"{fence}\n{value}\n{fence}", " ") def _inline_code(value): num_backticks = 1 @@ -2066,21 +2055,19 @@ def _inline_code(value): num_backticks += 1 delimiter = "`" * num_backticks if value.startswith("`") or value.endswith("`"): - return "{} {} {}".format(delimiter, value, delimiter) - return "{}{}{}".format(delimiter, value, delimiter) + return f"{delimiter} {value} {delimiter}" + return f"{delimiter}{value}{delimiter}" def _format_tool_call_arguments(arguments): if not isinstance(arguments, dict) or not arguments: - return " Arguments: {}".format(_inline_code(json.dumps(arguments))) + return f" Arguments: {_inline_code(json.dumps(arguments))}" lines = [] for key, value in arguments.items(): if isinstance(value, str): - lines.append(" {}:".format(key)) + lines.append(f" {key}:") lines.append(_fenced_block(value)) else: - lines.append( - " {}: {}".format(key, _inline_code(json.dumps(value))) - ) + lines.append(f" {key}: {_inline_code(json.dumps(value))}") return "\n".join(lines) def _token_usage_markdown(input_tokens, output_tokens, token_details): @@ -2088,7 +2075,7 @@ def _token_usage_markdown(input_tokens, output_tokens, token_details): if token_details: details = _inline_code(json.dumps(token_details)) if usage: - return "{}, {}".format(usage, details) + return f"{usage}, {details}" return details return usage @@ -2207,9 +2194,9 @@ def _display_fragments(fragments, title): options = json.loads(options) if options: options_text = "\n".join( - "- {}: {}".format(key, value) for key, value in options.items() + f"- {key}: {value}" for key, value in options.items() ) - click.echo("\n## Options\n\n{}".format(options_text)) + click.echo(f"\n## Options\n\n{options_text}") if row["system"] != current_system: if row["system"] is not None: click.echo("\n## System\n\n{}".format(row["system"])) @@ -2255,7 +2242,7 @@ def _display_fragments(fragments, title): desc += attachment["url"] elif attachment.get("content"): desc += f"<{attachment['content_length']:,} bytes>" - attachments += "\n - {}".format(desc) + attachments += f"\n - {desc}" click.echo( "- **{}**: `{}`
\n{}{}{}".format( tool_result["name"], @@ -2300,7 +2287,7 @@ def _display_fragments(fragments, title): if row["schema_json"]: try: parsed = json.loads(response) - response = "```json\n{}\n```".format(json.dumps(parsed, indent=2)) + response = f"```json\n{json.dumps(parsed, indent=2)}\n```" except ValueError: pass if row.get("reasoning"): @@ -2318,7 +2305,7 @@ def _display_fragments(fragments, title): ) click.echo("") if response: - click.echo("{}\n".format(response)) + click.echo(f"{response}\n") if usage: token_usage = _token_usage_markdown( row["input_tokens"], @@ -2326,7 +2313,7 @@ def _display_fragments(fragments, title): json.loads(row["token_details"]) if row["token_details"] else None, ) if token_usage: - click.echo("## Token usage\n\n{}\n".format(token_usage)) + click.echo(f"## Token usage\n\n{token_usage}\n") @cli.group( @@ -2400,7 +2387,7 @@ def render_model_with_aliases( initial_indent=" ", subsequent_indent=" ", ) - output += "\n Attachment types:\n{}".format(wrapper.fill(attachment_types)) + output += f"\n Attachment types:\n{wrapper.fill(attachment_types)}" features = ( [] + (["streaming"] if model.can_stream else []) @@ -2410,14 +2397,14 @@ def render_model_with_aliases( ) if options and features: output += "\n Features:\n{}".format( - "\n".join(" - {}".format(feature) for feature in features) + "\n".join(f" - {feature}" for feature in features) ) if options and hasattr(model, "needs_key") and model.needs_key: output += "\n Keys:" if hasattr(model, "needs_key") and model.needs_key: - output += "\n key: {}".format(model.needs_key) + output += f"\n key: {model.needs_key}" if hasattr(model, "key_env_var") and model.key_env_var: - output += "\n env_var: {}".format(model.key_env_var) + output += f"\n env_var: {model.key_env_var}" return output @@ -2430,7 +2417,7 @@ def render_model_with_options(model_id, *, async_=False): async_=async_, models_that_have_shown_options=set(), ) - raise click.ClickException("'{}' is not a known model".format(model_id)) + raise click.ClickException(f"'{model_id}' is not a known model") @models.command(name="list") @@ -2453,13 +2440,11 @@ def models_list(options, async_, schemas, tools, query, model_ids): for model_with_aliases in get_models_with_aliases(): if async_ and not model_with_aliases.async_model: continue - if query: - # Only show models where every provided query string matches - if not all(model_with_aliases.matches(q) for q in query): - continue - if model_ids: - if not model_matches_id_or_alias(model_with_aliases, model_ids): - continue + # Only show models where every provided query string matches + if query and not all(model_with_aliases.matches(q) for q in query): + continue + if model_ids and not model_matches_id_or_alias(model_with_aliases, model_ids): + continue if schemas and not model_with_aliases.model.supports_schema: continue if tools and not model_with_aliases.model.supports_tools: @@ -2488,7 +2473,7 @@ def models_default(model): model = get_model(model) set_default_model(model.model_id) except KeyError: - raise click.ClickException("Unknown model: {}".format(model)) + raise click.ClickException(f"Unknown model: {model}") @cli.group( @@ -2541,7 +2526,7 @@ def templates_show(name): raise click.ClickException(f"Template '{name}' not found or invalid") click.echo( yaml.dump( - dict((k, v) for k, v in template.model_dump().items() if v is not None), + {k: v for k, v in template.model_dump().items() if v is not None}, indent=4, default_flow_style=False, ) @@ -2621,7 +2606,7 @@ def schemas_list(path, database, queries, full, json_, nl): path = database path = pathlib.Path(path or logs_db_path()) if not path.exists(): - raise click.ClickException("No log database found at {}".format(path)) + raise click.ClickException(f"No log database found at {path}") db = sqlite_utils.Database(path) migrate(db) @@ -2630,9 +2615,9 @@ def schemas_list(path, database, queries, full, json_, nl): if queries: where_bits = ["schemas.content like ?" for _ in queries] where_sql += " where {}".format(" and ".join(where_bits)) - params.extend("%{}%".format(q) for q in queries) + params.extend(f"%{q}%" for q in queries) - sql = """ + sql = f""" select schemas.id, schemas.content, @@ -2641,9 +2626,9 @@ def schemas_list(path, database, queries, full, json_, nl): from schemas join responses on responses.schema_id = schemas.id - {} group by responses.schema_id + {where_sql} group by responses.schema_id order by recently_used - """.format(where_sql) + """ rows = db.query(sql, params) if json_ or nl: @@ -2697,7 +2682,7 @@ def schemas_show(schema_id, path, database): path = database path = pathlib.Path(path or logs_db_path()) if not path.exists(): - raise click.ClickException("No log database found at {}".format(path)) + raise click.ClickException(f"No log database found at {path}") db = sqlite_utils.Database(path) migrate(db) @@ -2816,7 +2801,7 @@ def introspect_tools(toolbox_class): "{}{}{}\n".format( tool.name, sig, - " (plugin: {})".format(tool.plugin) if tool.plugin else "", + f" (plugin: {tool.plugin})" if tool.plugin else "", ) ) if tool.description: @@ -2829,12 +2814,7 @@ def introspect_tools(toolbox_class): .replace("(self, ", "(") .replace("(self)", "()") ) - click.echo( - " {}{}\n".format( - tool.name, - sig, - ) - ) + click.echo(f" {tool.name}{sig}\n") if tool.description: click.echo(textwrap.indent(tool.description.strip(), " ") + "\n") @@ -2974,12 +2954,10 @@ def fragments_list(queries, aliases, json_): db = sqlite_utils.Database(logs_db_path()) migrate(db) params = {} - param_count = 0 where_bits = [] if aliases: where_bits.append("fragment_aliases.alias is not null") - for q in queries: - param_count += 1 + for param_count, q in enumerate(queries, start=1): p = f"p{param_count}" params[p] = q where_bits.append(f""" @@ -2990,7 +2968,7 @@ def fragments_list(queries, aliases, json_): where = "\n and\n ".join(where_bits) if where: where = " where " + where - sql = """ + sql = f""" select fragments.hash, json_group_array(fragment_aliases.alias) filter ( @@ -3008,7 +2986,7 @@ def fragments_list(queries, aliases, json_): group by fragments.id, fragments.hash, fragments.content, fragments.datetime_utc, fragments.source order by fragments.datetime_utc - """.format(where=where) + """ results = list(db.query(sql, params)) for result in results: result["aliases"] = json.loads(result["aliases"]) @@ -3417,11 +3395,8 @@ def embed_multi( if not input_path and not sql and not files: raise click.UsageError("Either --sql or input path or --files is required") - if files: - if input_path or sql or format: - raise click.UsageError( - "Cannot use --files with --sql, input path or --format" - ) + if files and (input_path or sql or format): + raise click.UsageError("Cannot use --files with --sql, input path or --format") if database: db = sqlite_utils.Database(database) @@ -3473,7 +3448,7 @@ def iterate_files(): if content is None: # Log to stderr click.echo( - "Could not decode text in file {}".format(path), + f"Could not decode text in file {path}", err=True, ) else: @@ -3483,7 +3458,7 @@ def iterate_files(): rows = iterate_files() elif sql: rows = db.query(sql) - count_sql = "select count(*) as c from ({})".format(sql) + count_sql = f"select count(*) as c from ({sql})" expected_length = next(db.query(count_sql))["c"] else: @@ -3498,11 +3473,15 @@ def load_rows(fp): for _ in load_rows(fp): expected_length += 1 - rows = load_rows( - open(input_path, "rb") - if input_path != "-" - else io.BufferedReader(sys.stdin.buffer) - ) + if input_path != "-": + + def rows_from_input(): + with open(input_path, "rb") as fp: + yield from load_rows(fp) + + rows = rows_from_input() + else: + rows = load_rows(io.BufferedReader(sys.stdin.buffer)) except json.JSONDecodeError as ex: raise click.ClickException(str(ex)) @@ -3510,11 +3489,11 @@ def load_rows(fp): rows, label="Embedding", show_percent=True, length=expected_length ) as rows: - def tuples() -> Iterable[Tuple[str, Union[bytes, str]]]: + def tuples() -> Iterable[tuple[str, bytes | str]]: for row in rows: values = list(row.values()) id: str = prefix + str(values[0]) - content: Optional[Union[bytes, str]] = None + content: bytes | str | None = None if binary: content = cast(bytes, values[1]) else: @@ -3633,9 +3612,8 @@ def embed_models_list(query): "List available embedding models" output = [] for model_with_aliases in get_embedding_models_with_aliases(): - if query: - if not all(model_with_aliases.matches(q) for q in query): - continue + if query and not all(model_with_aliases.matches(q) for q in query): + continue s = str(model_with_aliases.model) if model_with_aliases.aliases: s += " (aliases: {})".format(", ".join(model_with_aliases.aliases)) @@ -3665,7 +3643,7 @@ def embed_models_default(model, remove_default): model = get_embedding_model(model) set_default_embedding_model(model.model_id) except KeyError: - raise click.ClickException("Unknown embedding model: {}".format(model)) + raise click.ClickException(f"Unknown embedding model: {model}") @cli.group( @@ -3697,7 +3675,7 @@ def embed_db_collections(database, json_): database = database or (user_dir() / "embeddings.db") db = sqlite_utils.Database(str(database)) if not db["collections"].exists(): - raise click.ClickException("No collections table found in {}".format(database)) + raise click.ClickException(f"No collections table found in {database}") rows = db.query(""" select collections.name, @@ -3938,7 +3916,7 @@ def _human_readable_size(size_bytes): size_bytes /= 1024.0 i += 1 - return "{:.2f}{}".format(size_bytes, size_name[i]) + return f"{size_bytes:.2f}{size_name[i]}" def logs_on(): @@ -4048,7 +4026,7 @@ def _parse_yaml_template(name, content): try: loaded = yaml.safe_load(content) except yaml.YAMLError as ex: - raise LoadTemplateError("Invalid YAML: {}".format(str(ex))) + raise LoadTemplateError(f"Invalid YAML: {ex!s}") if isinstance(loaded, str): return Template(name=name, prompt=loaded) loaded["name"] = name @@ -4062,12 +4040,12 @@ def _parse_yaml_template(name, content): def load_template(name: str) -> Template: "Load template, or raise LoadTemplateError(msg)" - if name.startswith("https://") or name.startswith("http://"): + if name.startswith(("https://", "http://")): response = httpx.get(name) try: response.raise_for_status() except httpx.HTTPStatusError as ex: - raise LoadTemplateError("Could not load template {}: {}".format(name, ex)) + raise LoadTemplateError(f"Could not load template {name}: {ex}") return _parse_yaml_template(name, response.text) potential_path = pathlib.Path(name) @@ -4076,12 +4054,12 @@ def load_template(name: str) -> Template: prefix, rest = name.split(":", 1) loaders = get_template_loaders() if prefix not in loaders: - raise LoadTemplateError("Unknown template prefix: {}".format(prefix)) + raise LoadTemplateError(f"Unknown template prefix: {prefix}") loader = loaders[prefix] try: return loader(rest) - except Exception as ex: - raise LoadTemplateError("Could not load template {}: {}".format(name, ex)) + except Exception as ex: # noqa: BLE001 + raise LoadTemplateError(f"Could not load template {name}: {ex}") # Try local file if potential_path.exists(): @@ -4098,7 +4076,7 @@ def load_template(name: str) -> Template: return template_obj -def _tools_from_code(code_or_path: str) -> List[Tool]: +def _tools_from_code(code_or_path: str) -> list[Tool]: """ Treat all Python functions in the code as tools """ @@ -4106,13 +4084,13 @@ def _tools_from_code(code_or_path: str) -> List[Tool]: try: code_or_path = pathlib.Path(code_or_path).read_text() except FileNotFoundError: - raise click.ClickException("File not found: {}".format(code_or_path)) - namespace: Dict[str, Any] = {} + raise click.ClickException(f"File not found: {code_or_path}") + namespace: dict[str, Any] = {} tools = [] try: - exec(code_or_path, namespace) + exec(code_or_path, namespace) # noqa: S102 except SyntaxError as ex: - raise click.ClickException("Error in --functions definition: {}".format(ex)) + raise click.ClickException(f"Error in --functions definition: {ex}") # Register all callables in the locals dict: for name, value in namespace.items(): if callable(value) and not name.startswith("_"): @@ -4123,7 +4101,7 @@ def _tools_from_code(code_or_path: str) -> List[Tool]: def _debug_tool_call(_, tool_call, tool_result): click.echo( click.style( - "\nTool call: {}({})".format(tool_call.name, tool_call.arguments), + f"\nTool call: {tool_call.name}({tool_call.arguments})", fg="yellow", bold=True, ), @@ -4134,7 +4112,7 @@ def _debug_tool_call(_, tool_call, tool_result): if tool_result.attachments: attachments += "\nAttachments:\n" for attachment in tool_result.attachments: - attachments += f" {repr(attachment)}\n" + attachments += f" {attachment!r}\n" try: output = json.dumps(json.loads(tool_result.output), indent=2) @@ -4152,7 +4130,7 @@ def _debug_tool_call(_, tool_call, tool_result): if tool_result.exception: click.echo( click.style( - " Exception: {}".format(tool_result.exception), + f" Exception: {tool_result.exception}", fg="red", bold=True, ), @@ -4163,7 +4141,7 @@ def _debug_tool_call(_, tool_call, tool_result): def _approve_tool_call(_, tool_call): click.echo( click.style( - "Tool call: {}({})".format(tool_call.name, tool_call.arguments), + f"Tool call: {tool_call.name}({tool_call.arguments})", fg="yellow", bold=True, ), @@ -4174,18 +4152,16 @@ def _approve_tool_call(_, tool_call): def _gather_tools( - tool_specs: List[str], python_tools: List[str] -) -> List[Union[Tool, Type[Toolbox]]]: - tools: List[Union[Tool, Type[Toolbox]]] = [] + tool_specs: list[str], python_tools: list[str] +) -> list[Tool | type[Toolbox]]: + tools: list[Tool | type[Toolbox]] = [] if python_tools: for code_or_path in python_tools: tools.extend(_tools_from_code(code_or_path)) registered_tools = get_tools() - registered_classes = dict( - (key, value) - for key, value in registered_tools.items() - if inspect.isclass(value) - ) + registered_classes = { + key: value for key, value in registered_tools.items() if inspect.isclass(value) + } bad_tools = [ tool for tool in tool_specs if tool.split("(")[0] not in registered_tools ] diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 79f723f18..30583150f 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1,3 +1,17 @@ +import datetime +import json +import os +from collections.abc import AsyncGenerator, Iterable, Iterator +from enum import Enum +from typing import Any, cast + +import click +import httpx +import openai +import yaml +from pydantic import Field, create_model, field_validator + +import llm from llm import ( AsyncConversation, AsyncKeyModel, @@ -9,36 +23,13 @@ Response, hookimpl, ) -import llm from llm.parts import StreamEvent from llm.utils import ( dicts_to_table_string, - remove_dict_none_values, logging_client, + remove_dict_none_values, simplify_usage_dict, ) -import click -import datetime -from enum import Enum -import httpx -import openai -import os - -from pydantic import create_model, field_validator, Field - -from typing import ( - Any, - AsyncGenerator, - cast, - Dict, - List, - Iterable, - Iterator, - Optional, - Union, -) -import json -import yaml @hookimpl @@ -396,7 +387,7 @@ def __init__(self, model_id, openai_model_id, dimensions=None): self.openai_model_id = openai_model_id self.dimensions = dimensions - def embed_batch(self, items: Iterable[Union[str, bytes]]) -> Iterator[List[float]]: + def embed_batch(self, items: Iterable[str | bytes]) -> Iterator[list[float]]: kwargs = { "input": items, "model": self.openai_model_id, @@ -447,12 +438,12 @@ def models(json_, key): "created": created_str, } ) - done = dicts_to_table_string("id owned_by created".split(), to_print) + done = dicts_to_table_string(["id", "owned_by", "created"], to_print) print("\n".join(done)) class SharedOptions(llm.Options): - temperature: Optional[float] = Field( + temperature: float | None = Field( description=( "What sampling temperature to use, between 0 and 2. Higher values like " "0.8 will make the output more random, while lower values like 0.2 will " @@ -462,10 +453,10 @@ class SharedOptions(llm.Options): le=2, default=None, ) - max_tokens: Optional[int] = Field( + max_tokens: int | None = Field( description="Maximum number of tokens to generate.", default=None ) - top_p: Optional[float] = Field( + top_p: float | None = Field( description=( "An alternative to sampling with temperature, called nucleus sampling, " "where the model considers the results of the tokens with top_p " @@ -477,7 +468,7 @@ class SharedOptions(llm.Options): le=1, default=None, ) - frequency_penalty: Optional[float] = Field( + frequency_penalty: float | None = Field( description=( "Number between -2.0 and 2.0. Positive values penalize new tokens based " "on their existing frequency in the text so far, decreasing the model's " @@ -487,7 +478,7 @@ class SharedOptions(llm.Options): le=2, default=None, ) - presence_penalty: Optional[float] = Field( + presence_penalty: float | None = Field( description=( "Number between -2.0 and 2.0. Positive values penalize new tokens based " "on whether they appear in the text so far, increasing the model's " @@ -497,18 +488,18 @@ class SharedOptions(llm.Options): le=2, default=None, ) - stop: Optional[str] = Field( + stop: str | None = Field( description=("A string where the API will stop generating further tokens."), default=None, ) - logit_bias: Optional[Union[dict, str]] = Field( + logit_bias: dict | str | None = Field( description=( "Modify the likelihood of specified tokens appearing in the completion. " 'Pass a JSON string like \'{"1712":-100, "892":-100, "1489":-100}\'' ), default=None, ) - seed: Optional[int] = Field( + seed: int | None = Field( description="Integer seed to attempt to sample deterministically", default=None, ) @@ -584,7 +575,7 @@ def build_options_class( ): fields = { "json_object": ( - Optional[bool], + bool | None, Field( description="Output a valid JSON object {...}. Prompt must mention JSON.", default=None, @@ -593,7 +584,7 @@ def build_options_class( } if chat_completions: fields["chat_completions"] = ( - Optional[bool], + bool | None, Field( description=( "Force the use of the older /v1/chat/completions endpoint " @@ -609,7 +600,7 @@ def build_options_class( ) image_detail_values = enum_values_sentence(image_detail_enum) fields["image_detail"] = ( - Optional[image_detail_enum], + image_detail_enum | None, Field( description=( "Controls the detail level for image attachments. Supported values are " @@ -620,7 +611,7 @@ def build_options_class( ) if reasoning: fields["reasoning_effort"] = ( - Optional[ReasoningEffortEnum], + ReasoningEffortEnum | None, Field( description=( "Constraints effort on reasoning for reasoning models. Currently " @@ -633,7 +624,7 @@ def build_options_class( ) if verbosity: fields["verbosity"] = ( - Optional[VerbosityEnum], + VerbosityEnum | None, Field( description=( "Controls how verbose the model's response should be. Supported " @@ -741,7 +732,7 @@ def __init__( ) def __str__(self) -> str: - return "OpenAI Chat: {}".format(self.model_id) + return f"OpenAI Chat: {self.model_id}" def _append_llm_message(self, out, message, current_system, image_detail=None): """Translate one llm.Message into one (or more) OpenAI message @@ -827,10 +818,10 @@ def _append_llm_message(self, out, message, current_system, image_detail=None): def build_messages(self, prompt, conversation, image_detail=None): """Translate prompt.messages into OpenAI's wire format.""" - messages: List[Dict[str, Any]] = [] + messages: list[dict[str, Any]] = [] if image_detail is not None: image_detail = image_detail.value - current_system: Optional[str] = None + current_system: str | None = None for msg in prompt.messages: current_system = self._append_llm_message( messages, msg, current_system, image_detail=image_detail @@ -915,9 +906,9 @@ def execute( prompt: Prompt, stream: bool, response: Response, - conversation: Optional[Conversation] = None, - key: Optional[str] = None, - ) -> Iterator[Union[str, StreamEvent]]: + conversation: Conversation | None = None, + key: str | None = None, + ) -> Iterator[str | StreamEvent]: if prompt.system and not self.allows_system_prompt: raise NotImplementedError("Model does not support system prompts") messages = self.build_messages( @@ -1033,9 +1024,9 @@ async def execute( prompt: Prompt, stream: bool, response: AsyncResponse, - conversation: Optional[AsyncConversation] = None, - key: Optional[str] = None, - ) -> AsyncGenerator[Union[str, StreamEvent], None]: + conversation: AsyncConversation | None = None, + key: str | None = None, + ) -> AsyncGenerator[str | StreamEvent, None]: if prompt.system and not self.allows_system_prompt: raise NotImplementedError("Model does not support system prompts") messages = self.build_messages( @@ -1166,30 +1157,30 @@ class _SharedResponses(_Shared): """Mixin that translates llm.Prompt into Responses API parameters.""" def __str__(self) -> str: - return "OpenAI Responses: {}".format(self.model_id) + return f"OpenAI Responses: {self.model_id}" def _delegate_chat_kwargs(self): """Return constructor kwargs that mirror this Responses model so we can build a sibling Chat / AsyncChat instance for the ``-o chat_completions 1`` opt-out path.""" - return dict( - model_id=self.model_id, - key=self.key, - model_name=self.model_name, - api_base=self.api_base, - api_type=self.api_type, - api_version=self.api_version, - api_engine=self.api_engine, - headers=self.headers, - can_stream=self.can_stream, - vision=self.vision, - reasoning=self._reasoning, - verbosity=self._verbosity, - image_detail_original=self._image_detail_original, - supports_schema=self.supports_schema, - supports_tools=self.supports_tools, - allows_system_prompt=self.allows_system_prompt, - ) + return { + "model_id": self.model_id, + "key": self.key, + "model_name": self.model_name, + "api_base": self.api_base, + "api_type": self.api_type, + "api_version": self.api_version, + "api_engine": self.api_engine, + "headers": self.headers, + "can_stream": self.can_stream, + "vision": self.vision, + "reasoning": self._reasoning, + "verbosity": self._verbosity, + "image_detail_original": self._image_detail_original, + "supports_schema": self.supports_schema, + "supports_tools": self.supports_tools, + "allows_system_prompt": self.allows_system_prompt, + } def _build_responses_input(self, prompt, image_detail=None): """Translate prompt.messages into a (input_items, instructions) tuple @@ -1207,8 +1198,8 @@ def _build_responses_input(self, prompt, image_detail=None): ToolResultPart, ) - items: List[Dict[str, Any]] = [] - instructions: Optional[str] = None + items: list[dict[str, Any]] = [] + instructions: str | None = None for msg in prompt.messages: if msg.role == "system": @@ -1217,11 +1208,11 @@ def _build_responses_input(self, prompt, image_detail=None): instructions = text continue - text_bits: List[str] = [] - attachment_items: List[Dict[str, Any]] = [] - tool_call_items: List[Dict[str, Any]] = [] - tool_result_items: List[Dict[str, Any]] = [] - reasoning_items: List[Dict[str, Any]] = [] + text_bits: list[str] = [] + attachment_items: list[dict[str, Any]] = [] + tool_call_items: list[dict[str, Any]] = [] + tool_result_items: list[dict[str, Any]] = [] + reasoning_items: list[dict[str, Any]] = [] for part in msg.parts: if isinstance(part, TextPart): @@ -1256,7 +1247,7 @@ def _build_responses_input(self, prompt, image_detail=None): if enc or rid: # Round-trip a previous reasoning item so the model # can pick up where it left off mid-tool-call. - item: Dict[str, Any] = {"type": "reasoning"} + item: dict[str, Any] = {"type": "reasoning"} if rid: item["id"] = rid if enc: @@ -1277,7 +1268,7 @@ def _build_responses_input(self, prompt, image_detail=None): if msg.role == "user": if attachment_items: - content: List[Dict[str, Any]] = [] + content: list[dict[str, Any]] = [] if text_bits: content.append( {"type": "input_text", "text": "".join(text_bits)} @@ -1308,7 +1299,7 @@ def _build_responses_kwargs(self, prompt, stream): top_p = opts.pop("top_p", None) seed = opts.pop("seed", None) - kwargs: Dict[str, Any] = {} + kwargs: dict[str, Any] = {} if max_tokens is None and self.default_max_tokens is not None: max_tokens = self.default_max_tokens if max_tokens is not None: @@ -1328,7 +1319,7 @@ def _build_responses_kwargs(self, prompt, stream): if reasoning: kwargs["reasoning"] = reasoning - text: Dict[str, Any] = {} + text: dict[str, Any] = {} if verbosity: text["verbosity"] = verbosity if prompt.options.json_object: @@ -1400,7 +1391,7 @@ def _reasoning_event(self, item, *, include_text=True): enc = getattr(item, "encrypted_content", None) summary = getattr(item, "summary", None) text = self._reasoning_text_from_item(item) if include_text else "" - meta: Dict[str, Any] = {} + meta: dict[str, Any] = {} if rid: meta["id"] = rid if enc: @@ -1413,7 +1404,7 @@ def _reasoning_event(self, item, *, include_text=True): s.model_dump() if hasattr(s, "model_dump") else dict(s) for s in summary ] - except Exception: + except Exception: # noqa: BLE001 meta["summary"] = list(summary) return StreamEvent( type="reasoning", @@ -1484,9 +1475,9 @@ def execute( prompt: Prompt, stream: bool, response: Response, - conversation: Optional[Conversation] = None, - key: Optional[str] = None, - ) -> Iterator[Union[str, StreamEvent]]: + conversation: Conversation | None = None, + key: str | None = None, + ) -> Iterator[str | StreamEvent]: if getattr(prompt.options, "chat_completions", None): chat = Chat(**self._delegate_chat_kwargs()) yield from chat.execute(prompt, stream, response, conversation, key) @@ -1518,8 +1509,8 @@ def execute( stream=True, **kwargs, ) - tool_call_meta: Dict[str, Dict[str, str]] = {} - final_response_dict: Optional[Dict[str, Any]] = None + tool_call_meta: dict[str, dict[str, str]] = {} + final_response_dict: dict[str, Any] | None = None reasoning_items_with_streamed_text = set() for event in stream_obj: etype = getattr(event, "type", None) @@ -1711,9 +1702,9 @@ async def execute( prompt: Prompt, stream: bool, response: AsyncResponse, - conversation: Optional[AsyncConversation] = None, - key: Optional[str] = None, - ) -> AsyncGenerator[Union[str, StreamEvent], None]: + conversation: AsyncConversation | None = None, + key: str | None = None, + ) -> AsyncGenerator[str | StreamEvent, None]: if getattr(prompt.options, "chat_completions", None): chat = AsyncChat(**self._delegate_chat_kwargs()) async for event in chat.execute( @@ -1748,8 +1739,8 @@ async def execute( stream=True, **kwargs, ) - tool_call_meta: Dict[str, Dict[str, str]] = {} - final_response_dict: Optional[Dict[str, Any]] = None + tool_call_meta: dict[str, dict[str, str]] = {} + final_response_dict: dict[str, Any] | None = None reasoning_items_with_streamed_text = set() async for event in stream_obj: etype = getattr(event, "type", None) @@ -1881,7 +1872,7 @@ async def execute( class Completion(Chat): class Options(SharedOptions): - logprobs: Optional[int] = Field( + logprobs: int | None = Field( description="Include the log probabilities of most likely N per token", default=None, le=5, @@ -1892,16 +1883,16 @@ def __init__(self, *args, default_max_tokens=None, **kwargs): self.default_max_tokens = default_max_tokens def __str__(self) -> str: - return "OpenAI Completion: {}".format(self.model_id) + return f"OpenAI Completion: {self.model_id}" def execute( self, prompt: Prompt, stream: bool, response: Response, - conversation: Optional[Conversation] = None, - key: Optional[str] = None, - ) -> Iterator[Union[str, StreamEvent]]: + conversation: Conversation | None = None, + key: str | None = None, + ) -> Iterator[str | StreamEvent]: if prompt.system: raise NotImplementedError( "System prompts are not supported for OpenAI completion models" @@ -1949,7 +1940,7 @@ def not_nulls(data) -> dict: return {key: value for key, value in data if value is not None} -def combine_chunks(chunks: List) -> dict: +def combine_chunks(chunks: list) -> dict: content = "" role = None finish_reason = None diff --git a/llm/embeddings.py b/llm/embeddings.py index 90b983a11..c044b9253 100644 --- a/llm/embeddings.py +++ b/llm/embeddings.py @@ -1,21 +1,24 @@ -from .models import EmbeddingModel -from .embeddings_migrations import embeddings_migrations -from dataclasses import dataclass import hashlib -from itertools import islice import json +import time +from collections.abc import Iterable +from dataclasses import dataclass +from itertools import islice +from typing import Any, cast + from sqlite_utils import Database from sqlite_utils.db import Table -import time -from typing import cast, Any, Dict, Iterable, List, Optional, Tuple, Union + +from .embeddings_migrations import embeddings_migrations +from .models import EmbeddingModel @dataclass class Entry: id: str - score: Optional[float] - content: Optional[str] = None - metadata: Optional[Dict[str, Any]] = None + score: float | None + content: str | None = None + metadata: dict[str, Any] | None = None class Collection: @@ -25,10 +28,10 @@ class DoesNotExist(Exception): def __init__( self, name: str, - db: Optional[Database] = None, + db: Database | None = None, *, - model: Optional[EmbeddingModel] = None, - model_id: Optional[str] = None, + model: EmbeddingModel | None = None, + model_id: str | None = None, create: bool = True, ) -> None: """ @@ -115,8 +118,8 @@ def count(self) -> int: def embed( self, id: str, - value: Union[str, bytes], - metadata: Optional[Dict[str, Any]] = None, + value: str | bytes, + metadata: dict[str, Any] | None = None, store: bool = False, ) -> None: """ @@ -152,7 +155,7 @@ def embed( def embed_multi( self, - entries: Iterable[Tuple[str, Union[str, bytes]]], + entries: Iterable[tuple[str, str | bytes]], store: bool = False, batch_size: int = 100, ) -> None: @@ -172,7 +175,7 @@ def embed_multi( def embed_multi_with_metadata( self, - entries: Iterable[Tuple[str, Union[str, bytes], Optional[Dict[str, Any]]]], + entries: Iterable[tuple[str, str | bytes, dict[str, Any] | None]], store: bool = False, batch_size: int = 100, ) -> None: @@ -237,11 +240,11 @@ def embed_multi_with_metadata( def similar_by_vector( self, - vector: List[float], + vector: list[float], number: int = 10, - skip_id: Optional[str] = None, - prefix: Optional[str] = None, - ) -> List[Entry]: + skip_id: str | None = None, + prefix: str | None = None, + ) -> list[Entry]: """ Find similar items in the collection by a given vector. @@ -295,8 +298,8 @@ def distance_score(other_encoded): ] def similar_by_id( - self, id: str, number: int = 10, prefix: Optional[str] = None - ) -> List[Entry]: + self, id: str, number: int = 10, prefix: str | None = None + ) -> list[Entry]: """ Find similar items in the collection by a given ID. @@ -324,8 +327,8 @@ def similar_by_id( ) def similar( - self, value: Union[str, bytes], number: int = 10, prefix: Optional[str] = None - ) -> List[Entry]: + self, value: str | bytes, number: int = 10, prefix: str | None = None + ) -> list[Entry]: """ Find similar items in the collection by a given value. @@ -360,7 +363,7 @@ def delete(self): self.db.execute("delete from collections where id = ?", [self.id]) @staticmethod - def content_hash(input: Union[str, bytes]) -> bytes: + def content_hash(input: str | bytes) -> bytes: "Hash content for deduplication. Override to change hashing behavior." if isinstance(input, str): input = input.encode("utf8") diff --git a/llm/embeddings_migrations.py b/llm/embeddings_migrations.py index 96444bd65..678aa4496 100644 --- a/llm/embeddings_migrations.py +++ b/llm/embeddings_migrations.py @@ -1,7 +1,8 @@ -from sqlite_migrate import Migrations import hashlib import time +from sqlite_migrate import Migrations + embeddings_migrations = Migrations("llm.embeddings") diff --git a/llm/hookspecs.py b/llm/hookspecs.py index 7ab555199..0cf46b546 100644 --- a/llm/hookspecs.py +++ b/llm/hookspecs.py @@ -1,5 +1,4 @@ -from pluggy import HookimplMarker -from pluggy import HookspecMarker +from pluggy import HookimplMarker, HookspecMarker hookspec = HookspecMarker("llm") hookimpl = HookimplMarker("llm") diff --git a/llm/migrations.py b/llm/migrations.py index 985aaa62f..b93e83014 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -1,7 +1,7 @@ import datetime -from typing import Callable, List +from collections.abc import Callable -MIGRATIONS: List[Callable] = [] +MIGRATIONS: list[Callable] = [] migration = MIGRATIONS.append diff --git a/llm/models.py b/llm/models.py index 80fb6b1a2..ff58d6366 100644 --- a/llm/models.py +++ b/llm/models.py @@ -1,52 +1,56 @@ import asyncio import base64 -from condense_json import condense_json import dataclasses -from dataclasses import dataclass, field import datetime -from .errors import NeedsKeyException import hashlib -import httpx -from itertools import islice -from pathlib import Path import re import time -from types import MethodType -from typing import ( - TYPE_CHECKING, - Any, +from collections.abc import ( AsyncGenerator, AsyncIterator, Awaitable, Callable, - Dict, Iterable, Iterator, - List, +) +from dataclasses import dataclass, field +from itertools import islice +from pathlib import Path +from types import MethodType +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, Optional, - Set, Union, cast, get_type_hints, ) + +import httpx +from condense_json import condense_json + +from .errors import NeedsKeyException from .serialization import ResponseDict if TYPE_CHECKING: from .parts import StreamEvent +import inspect +import json +from abc import ABC, abstractmethod + +from pydantic import BaseModel, ConfigDict, create_model + from .utils import ( + Fragment, ensure_fragment, ensure_tool, make_schema_id, mimetype_from_path, mimetype_from_string, - token_usage_string, monotonic_ulid, - Fragment, + token_usage_string, ) -from abc import ABC, abstractmethod -import inspect -import json -from pydantic import BaseModel, ConfigDict, create_model CONVERSATION_NAME_LENGTH = 32 @@ -55,20 +59,20 @@ class Usage: "Token usage information from a model response." - input: Optional[int] = None - output: Optional[int] = None - details: Optional[Dict[str, Any]] = None + input: int | None = None + output: int | None = None + details: dict[str, Any] | None = None @dataclass class Attachment: "An attachment (image, audio, etc) to include with a prompt." - type: Optional[str] = None - path: Optional[str] = None - url: Optional[str] = None - content: Optional[bytes] = None - _id: Optional[str] = None + type: str | None = None + path: str | None = None + url: str | None = None + content: bytes | None = None + _id: str | None = None def id(self): # Hash of the binary content, or of '{"url": "https://..."}' for URL attachments @@ -142,10 +146,10 @@ class Tool: "A tool that can be called by a model." name: str - description: Optional[str] = None - input_schema: Dict = field(default_factory=dict) - implementation: Optional[Callable] = None - plugin: Optional[str] = None # plugin tool came from, e.g. 'llm_tools_sqlite' + description: str | None = None + input_schema: dict = field(default_factory=dict) + implementation: Callable | None = None + plugin: str | None = None # plugin tool came from, e.g. 'llm_tools_sqlite' def __post_init__(self): # Convert Pydantic model to JSON schema if needed @@ -226,8 +230,8 @@ def _implementation_arguments(tool: "Tool", tool_call: "ToolCall") -> dict: class Toolbox: - name: Optional[str] = None - instance_id: Optional[int] = None + name: str | None = None + instance_id: int | None = None _blocked = ( "tools", "add_tool", @@ -236,8 +240,8 @@ class Toolbox: "prepare", "prepare_async", ) - _extra_tools: List[Tool] = [] - _config: Dict[str, Any] = {} + _extra_tools: ClassVar[list[Tool]] = [] + _config: ClassVar[dict[str, Any]] = {} _prepared: bool = False _async_prepared: bool = False @@ -267,7 +271,7 @@ def wrapped_init(self, *args, **kwargs): cls.__init__ = wrapped_init @classmethod - def method_tools(cls) -> List[Tool]: + def method_tools(cls) -> list[Tool]: tools = [] for method_name in dir(cls): if method_name.startswith("_") or method_name in cls._blocked: @@ -276,7 +280,7 @@ def method_tools(cls) -> List[Tool]: if callable(method): tool = Tool.function( method, - name="{}_{}".format(cls.__name__, method_name), + name=f"{cls.__name__}_{method_name}", ) tools.append(tool) return tools @@ -295,7 +299,7 @@ def tools(self) -> Iterable[Tool]: yield from self._extra_tools def add_tool( - self, tool_or_function: Union[Tool, Callable[..., Any]], pass_self: bool = False + self, tool_or_function: Tool | Callable[..., Any], pass_self: bool = False ): "Add a tool to this toolbox" @@ -309,20 +313,18 @@ def _upgrade(fn): elif callable(tool_or_function): self._extra_tools.append(Tool.function(_upgrade(tool_or_function))) else: - raise ValueError("Tool must be an instance of Tool or a callable function") + raise TypeError("Tool must be an instance of Tool or a callable function") def prepare(self): """ Over-ride this to perform setup (and .add_tool() calls) before the toolbox is used. Implement a similar prepare_async() method for async setup. """ - pass async def prepare_async(self): """ Over-ride this to perform async setup (and .add_tool() calls) before the toolbox is used. """ - pass @dataclass @@ -331,7 +333,7 @@ class ToolCall: name: str arguments: dict - tool_call_id: Optional[str] = None + tool_call_id: str | None = None @dataclass @@ -340,25 +342,25 @@ class ToolResult: name: str output: str - attachments: List[Attachment] = field(default_factory=list) - tool_call_id: Optional[str] = None - instance: Optional[Toolbox] = None - exception: Optional[Exception] = None + attachments: list[Attachment] = field(default_factory=list) + tool_call_id: str | None = None + instance: Toolbox | None = None + exception: Exception | None = None @dataclass class ToolOutput: "Tool functions can return output with extra attachments" - output: Optional[Union[str, dict, list, bool, int, float]] = None - attachments: List[Attachment] = field(default_factory=list) + output: str | dict | list | bool | int | float | None = None + attachments: list[Attachment] = field(default_factory=list) -ToolDef = Union[Tool, Toolbox, Callable[..., Any]] -BeforeCallSync = Callable[[Optional[Tool], ToolCall], None] +ToolDef = Tool | Toolbox | Callable[..., Any] +BeforeCallSync = Callable[[Tool | None, ToolCall], None] AfterCallSync = Callable[[Tool, ToolCall, ToolResult], None] -BeforeCallAsync = Callable[[Optional[Tool], ToolCall], Union[None, Awaitable[None]]] -AfterCallAsync = Callable[[Tool, ToolCall, ToolResult], Union[None, Awaitable[None]]] +BeforeCallAsync = Callable[[Tool | None, ToolCall], None | Awaitable[None]] +AfterCallAsync = Callable[[Tool, ToolCall, ToolResult], None | Awaitable[None]] class CancelToolCall(Exception): @@ -387,24 +389,24 @@ class PauseChain(Exception): def __init__(self, *args): super().__init__(*args) - self.tool_call: Optional["ToolCall"] = None - self.tool_results: List["ToolResult"] = [] + self.tool_call: ToolCall | None = None + self.tool_results: list[ToolResult] = [] @dataclass class Prompt: "The prompt being sent to the model." - _prompt: Optional[str] + _prompt: str | None model: "Model" - fragments: Optional[List[Union[str, Fragment]]] - attachments: Optional[List[Attachment]] - _system: Optional[str] - system_fragments: Optional[List[Union[str, Fragment]]] - prompt_json: Optional[str] - schema: Optional[Union[Dict, type[BaseModel]]] - tools: List[Tool] - tool_results: List[ToolResult] + fragments: list[str | Fragment] | None + attachments: list[Attachment] | None + _system: str | None + system_fragments: list[str | Fragment] | None + prompt_json: str | None + schema: dict | type[BaseModel] | None + tools: list[Tool] + tool_results: list[ToolResult] options: "Options" hide_reasoning: bool @@ -486,7 +488,7 @@ def messages(self): if self._explicit_messages is not None: return list(self._explicit_messages) - result: List["Message"] = [] + result: list[Message] = [] if self.system: result.append(Message(role="system", parts=[TextPart(text=self.system)])) @@ -506,7 +508,7 @@ def messages(self): ) ) - user_parts: List[Any] = [] + user_parts: list[Any] = [] if self.prompt: user_parts.append(TextPart(text=self.prompt)) for att in self.attachments: @@ -517,7 +519,7 @@ def messages(self): return result -def _wrap_tools(tools: List[ToolDef]) -> List[Tool]: +def _wrap_tools(tools: list[ToolDef]) -> list[Tool]: wrapped_tools = [] for tool in tools: if isinstance(tool, Tool): @@ -527,7 +529,7 @@ def _wrap_tools(tools: List[ToolDef]) -> List[Tool]: elif callable(tool): wrapped_tools.append(Tool.function(tool)) else: - raise ValueError(f"Invalid tool: {tool}") + raise TypeError(f"Invalid tool: {tool}") return wrapped_tools @@ -541,14 +543,14 @@ def _combine_system(system, system_fragments): return "\n\n".join(bits) -def _merge_options(options: Optional[dict], kwargs: dict) -> dict: +def _merge_options(options: dict | None, kwargs: dict) -> dict: if not options: return kwargs overlap = set(options) & set(kwargs) if overlap: raise TypeError( "Got values for these options both in options= and as keyword " - "arguments: {}".format(sorted(overlap)) + f"arguments: {sorted(overlap)}" ) return {**options, **kwargs} @@ -557,10 +559,10 @@ def _merge_options(options: Optional[dict], kwargs: dict) -> dict: class _BaseConversation: model: "_BaseModel" id: str = field(default_factory=lambda: str(monotonic_ulid()).lower()) - name: Optional[str] = None - responses: List["_BaseResponse"] = field(default_factory=list) - tools: Optional[List[ToolDef]] = None - chain_limit: Optional[int] = None + name: str | None = None + responses: list["_BaseResponse"] = field(default_factory=list) + tools: list[ToolDef] | None = None + chain_limit: int | None = None @classmethod @abstractmethod @@ -569,13 +571,13 @@ def from_row(cls, row: Any) -> "_BaseConversation": def _build_full_chain( self, - prompt: Optional[str], + prompt: str | None, attachments, tool_results, explicit_messages, system=None, system_fragments=None, - ) -> List[Any]: + ) -> list[Any]: """Build the full message chain for the next turn. Uses the last response's stored prompt chain to recover prior @@ -599,7 +601,7 @@ def _build_full_chain( if explicit_messages is not None: return list(explicit_messages) - chain: List[Any] = [] + chain: list[Any] = [] if self.responses: last = self.responses[-1] # last.prompt.messages already contains the full input chain @@ -631,7 +633,7 @@ def _build_full_chain( ) ) - user_parts: List[Any] = [] + user_parts: list[Any] = [] if prompt: user_parts.append(TextPart(text=prompt)) for att in attachments or []: @@ -644,24 +646,24 @@ def _build_full_chain( @dataclass class Conversation(_BaseConversation): - before_call: Optional[BeforeCallSync] = None - after_call: Optional[AfterCallSync] = None + before_call: BeforeCallSync | None = None + after_call: AfterCallSync | None = None def prompt( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[Union[str, Fragment]]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - system_fragments: Optional[List[Union[str, Fragment]]] = None, - messages: Optional[List[Any]] = None, + fragments: list[str | Fragment] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + system_fragments: list[str | Fragment] | None = None, + messages: list[Any] | None = None, stream: bool = True, - key: Optional[str] = None, - options: Optional[dict] = None, + key: str | None = None, + options: dict | None = None, hide_reasoning: bool = False, **kwargs, ) -> "Response": @@ -699,22 +701,22 @@ def prompt( def chain( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[str]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - system_fragments: Optional[List[str]] = None, - messages: Optional[List[Any]] = None, + fragments: list[str] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + system_fragments: list[str] | None = None, + messages: list[Any] | None = None, stream: bool = True, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - chain_limit: Optional[int] = None, - before_call: Optional[BeforeCallSync] = None, - after_call: Optional[AfterCallSync] = None, - key: Optional[str] = None, - options: Optional[dict] = None, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + chain_limit: int | None = None, + before_call: BeforeCallSync | None = None, + after_call: AfterCallSync | None = None, + key: str | None = None, + options: dict | None = None, hide_reasoning: bool = False, ) -> "ChainResponse": self.model._validate_attachments(attachments) @@ -772,27 +774,27 @@ def __repr__(self): @dataclass class AsyncConversation(_BaseConversation): - before_call: Optional[BeforeCallAsync] = None - after_call: Optional[AfterCallAsync] = None + before_call: BeforeCallAsync | None = None + after_call: AfterCallAsync | None = None def chain( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[str]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - system_fragments: Optional[List[str]] = None, - messages: Optional[List[Any]] = None, + fragments: list[str] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + system_fragments: list[str] | None = None, + messages: list[Any] | None = None, stream: bool = True, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - chain_limit: Optional[int] = None, - before_call: Optional[BeforeCallAsync] = None, - after_call: Optional[AfterCallAsync] = None, - key: Optional[str] = None, - options: Optional[dict] = None, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + chain_limit: int | None = None, + before_call: BeforeCallAsync | None = None, + after_call: AfterCallAsync | None = None, + key: str | None = None, + options: dict | None = None, hide_reasoning: bool = False, ) -> "AsyncChainResponse": self.model._validate_attachments(attachments) @@ -830,19 +832,19 @@ def chain( def prompt( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[str]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - system_fragments: Optional[List[str]] = None, - messages: Optional[List[Any]] = None, + fragments: list[str] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + system_fragments: list[str] | None = None, + messages: list[Any] | None = None, stream: bool = True, - key: Optional[str] = None, - options: Optional[dict] = None, + key: str | None = None, + options: dict | None = None, hide_reasoning: bool = False, **kwargs, ) -> "AsyncResponse": @@ -928,18 +930,17 @@ class _BaseResponse: id: str prompt: "Prompt" stream: bool - resolved_model: Optional[str] = None + resolved_model: str | None = None conversation: Optional["_BaseConversation"] = None - _key: Optional[str] = None - _tool_calls: List[ToolCall] = [] + _key: str | None = None def __init__( self, prompt: Prompt, model: "_BaseModel", stream: bool, - conversation: Optional[_BaseConversation] = None, - key: Optional[str] = None, + conversation: _BaseConversation | None = None, + key: str | None = None, ): self.id = str(monotonic_ulid()).lower() self.prompt = prompt @@ -947,12 +948,12 @@ def __init__( self.model = model self.stream = stream self._key = key - self._chunks: List[str] = [] + self._chunks: list[str] = [] # Every StreamEvent ever yielded by execute(), in order. Plain # str yields are wrapped as text events (with part_index resolved # by _resolve_part_index) so this buffer is the single source of # truth for replay and for assembling response.messages. - self._stream_events: List[Any] = [] + self._stream_events: list[Any] = [] # Auto-allocator state for resolving StreamEvent.part_index=None. # Plugins yield events with part_index=None (the default) and # the framework assigns concrete integers based on context: @@ -964,21 +965,21 @@ def __init__( # share an index; _auto_tool_id_to_index maps known tool ids to # their assigned index for parallel-tool-call grouping. self._auto_index_max: int = -1 - self._auto_last_index: Optional[int] = None - self._auto_last_family: Optional[str] = None - self._auto_tool_id_to_index: Dict[str, int] = {} + self._auto_last_index: int | None = None + self._auto_last_family: str | None = None + self._auto_tool_id_to_index: dict[str, int] = {} self._done = False - self._tool_calls: List[ToolCall] = [] - self.response_json: Optional[Dict[str, Any]] = None + self._tool_calls: list[ToolCall] = [] + self.response_json: dict[str, Any] | None = None self.conversation = conversation - self.attachments: List[Attachment] = [] - self._start: Optional[float] = None - self._end: Optional[float] = None - self._start_utcnow: Optional[datetime.datetime] = None - self.input_tokens: Optional[int] = None - self.output_tokens: Optional[int] = None - self.token_details: Optional[dict] = None - self.done_callbacks: List[Callable] = [] + self.attachments: list[Attachment] = [] + self._start: float | None = None + self._end: float | None = None + self._start_utcnow: datetime.datetime | None = None + self.input_tokens: int | None = None + self.output_tokens: int | None = None + self.token_details: dict | None = None + self.done_callbacks: list[Callable] = [] if self.prompt.schema and not self.model.supports_schema: raise ValueError(f"{self.model} does not support schemas") @@ -986,7 +987,7 @@ def __init__( if self.prompt.tools and not self.model.supports_tools: raise ValueError(f"{self.model} does not support tools") - def _messages_now(self) -> List[Any]: + def _messages_now(self) -> list[Any]: """Assemble messages assuming the response is already drained. Public ``messages()`` forces / awaits first, then delegates here. @@ -1022,8 +1023,7 @@ def _resolve_part_index(self, event): fam = self._event_family(event.type) if event.part_index is not None: - if event.part_index > self._auto_index_max: - self._auto_index_max = event.part_index + self._auto_index_max = max(self._auto_index_max, event.part_index) if ( event.type in ("tool_call_name", "tool_call_args") and event.tool_call_id @@ -1109,7 +1109,7 @@ def _process_chunk(self, chunk): self._chunks.append(chunk) return chunk - def _build_parts(self) -> List[Any]: + def _build_parts(self) -> list[Any]: """Assemble Part objects from the accumulated stream events. Events sharing a part_index group into one Part. Mixing @@ -1137,7 +1137,7 @@ def _build_parts(self) -> List[Any]: # _tool_calls so response.messages isn't empty after # from_row, and Conversation.prompt-built chains include # the assistant turn on follow-up calls. - fallback_parts: List[Any] = [] + fallback_parts: list[Any] = [] text = "".join(self._chunks) if text: fallback_parts.append(TextPart(text=text)) @@ -1156,8 +1156,8 @@ def _build_parts(self) -> List[Any]: # per group. This handles non-adjacent same-index events (e.g. # text → tool_call → text where the plugin pinned both text # bursts to part_index=0) by merging them into one Part. - groups: Dict[int, List[Any]] = {} - order: List[int] = [] + groups: dict[int, list[Any]] = {} + order: list[int] = [] for event in self._stream_events: pi = event.part_index if pi not in groups: @@ -1165,7 +1165,7 @@ def _build_parts(self) -> List[Any]: order.append(pi) groups[pi].append(event) - parts: List[Any] = [] + parts: list[Any] = [] for pi in order: evs = groups[pi] fam_first = self._event_family(evs[0].type) @@ -1177,7 +1177,7 @@ def _build_parts(self) -> List[Any]: "Allocate a new part_index for a different content type." ) - pm_merged: Optional[Dict[str, Any]] = None + pm_merged: dict[str, Any] | None = None for e in evs: if e.provider_metadata: merged = dict(pm_merged) if pm_merged else {} @@ -1279,16 +1279,16 @@ def add_tool_call(self, tool_call: ToolCall): # state on a call) to invent fallback matching schemes. tool_call = dataclasses.replace( tool_call, - tool_call_id="tc_{}".format(str(monotonic_ulid()).lower()), + tool_call_id=f"tc_{str(monotonic_ulid()).lower()}", ) self._tool_calls.append(tool_call) def set_usage( self, *, - input: Optional[int] = None, - output: Optional[int] = None, - details: Optional[dict] = None, + input: int | None = None, + output: int | None = None, + details: dict | None = None, ): self.input_tokens = input self.output_tokens = output @@ -1299,7 +1299,7 @@ def set_resolved_model(self, model_id: str): @classmethod def from_row(cls, db, row, _async=False): - from llm import get_model, get_async_model + from llm import get_async_model, get_model if _async: model = get_async_model(row["model"]) @@ -1581,10 +1581,7 @@ def log_to_db(self, db): "instance_id": instance_id, "exception": ( ( - "{}: {}".format( - tool_result.exception.__class__.__name__, - str(tool_result.exception), - ) + f"{tool_result.exception.__class__.__name__}: {tool_result.exception!s}" ) if tool_result.exception else None @@ -1627,7 +1624,7 @@ def _response_to_dict(response: "_BaseResponse") -> ResponseDict: for key, value in dict(response.prompt.options).items() if value is not None } - payload: Dict[str, Any] = { + payload: dict[str, Any] = { "model": response.model.model_id, "prompt": { "messages": [m.to_dict() for m in response.prompt.messages], @@ -1643,7 +1640,7 @@ def _response_to_dict(response: "_BaseResponse") -> ResponseDict: payload["id"] = response.id if response._done: if response.input_tokens is not None or response.output_tokens is not None: - usage: Dict[str, Any] = {} + usage: dict[str, Any] = {} if response.input_tokens is not None: usage["input"] = response.input_tokens if response.output_tokens is not None: @@ -1722,11 +1719,11 @@ class Response(_BaseResponse): def reply( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - messages: Optional[List[Any]] = None, - tool_results: Optional[List[ToolResult]] = None, - options: Optional[dict] = None, + messages: list[Any] | None = None, + tool_results: list[ToolResult] | None = None, + options: dict | None = None, **kwargs, ) -> "Response": """Continue the conversation from this response. @@ -1751,7 +1748,7 @@ def reply( # (mirrors Conversation.prompt's `tools or self.tools` rule). if "tools" not in kwargs and self.prompt.tools: kwargs["tools"] = self.prompt.tools - chain: List[Any] = list(self.prompt.messages) + list(self._messages_now()) + chain: list[Any] = list(self.prompt.messages) + list(self._messages_now()) if tool_results: chain.append( Message( @@ -1833,10 +1830,10 @@ def text_or_raise(self) -> str: def execute_tool_calls( self, *, - before_call: Optional[BeforeCallSync] = None, - after_call: Optional[AfterCallSync] = None, - tool_calls_list: Optional[List[ToolCall]] = None, - ) -> List[ToolResult]: + before_call: BeforeCallSync | None = None, + after_call: AfterCallSync | None = None, + tool_calls_list: list[ToolCall] | None = None, + ) -> list[ToolResult]: """Execute tool calls using this response's tools. By default executes ``self.tool_calls()``; pass @@ -1860,7 +1857,7 @@ def execute_tool_calls( inst._prepared = True for tool_call in tool_calls_list: - tool: Optional[Tool] = tools_by_name.get(tool_call.name) + tool: Tool | None = tools_by_name.get(tool_call.name) # Tool could be None if the tool was not found in the prompt tools, # but we still call the before_call method: if before_call: @@ -1883,7 +1880,7 @@ def execute_tool_calls( continue if tool is None: - msg = 'tool "{}" does not exist'.format(tool_call.name) + msg = f'tool "{tool_call.name}" does not exist' tool_results.append( ToolResult( name=tool_call.name, @@ -1896,7 +1893,7 @@ def execute_tool_calls( if not tool.implementation: raise ValueError( - "No implementation available for tool: {}".format(tool_call.name) + f"No implementation available for tool: {tool_call.name}" ) attachments = [] @@ -1924,7 +1921,7 @@ def execute_tool_calls( ex.tool_call = tool_call ex.tool_results = list(tool_results) raise - except Exception as ex: + except Exception as ex: # noqa: BLE001 result = f"Error: {ex}" exception = ex @@ -1947,15 +1944,15 @@ def execute_tool_calls( tool_results.append(tool_result_obj) return tool_results - def tool_calls(self) -> List[ToolCall]: + def tool_calls(self) -> list[ToolCall]: "Return the list of tool calls made during this response." self._force() return self._tool_calls - def tool_calls_or_raise(self) -> List[ToolCall]: + def tool_calls_or_raise(self) -> list[ToolCall]: return self.tool_calls() - def json(self) -> Optional[Dict[str, Any]]: + def json(self) -> dict[str, Any] | None: "Return the raw JSON response from the model, if available." self._force() return self.response_json @@ -1997,7 +1994,7 @@ def _iter_events(self): key=self.model.get_key(self._key), ) else: - raise Exception("self.model must be a Model or KeyModel") + raise TypeError("self.model must be a Model or KeyModel") for chunk in generator: assert chunk is not None @@ -2047,7 +2044,7 @@ def stream_events(self): self._done = True self._on_done() - def messages(self) -> List[Any]: + def messages(self) -> list[Any]: """List of Message objects produced by this response. Almost always a single assistant Message; multiple messages are @@ -2068,7 +2065,7 @@ def __repr__(self): text = "... not yet done ..." if self._done: text = "".join(self._chunks) - return "".format(self.prompt.prompt, text) + return f"" class AsyncResponse(_BaseResponse): @@ -2079,11 +2076,11 @@ class AsyncResponse(_BaseResponse): async def reply( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - messages: Optional[List[Any]] = None, - tool_results: Optional[List[ToolResult]] = None, - options: Optional[dict] = None, + messages: list[Any] | None = None, + tool_results: list[ToolResult] | None = None, + options: dict | None = None, **kwargs, ) -> "AsyncResponse": """Async counterpart of Response.reply(). Requires this response @@ -2103,7 +2100,7 @@ async def reply( tool_results = await self.execute_tool_calls() if "tools" not in kwargs and self.prompt.tools: kwargs["tools"] = self.prompt.tools - chain: List[Any] = list(self.prompt.messages) + list(self._messages_now()) + chain: list[Any] = list(self.prompt.messages) + list(self._messages_now()) if tool_results: chain.append( Message( @@ -2173,10 +2170,10 @@ async def _on_done(self): async def execute_tool_calls( self, *, - before_call: Optional[BeforeCallAsync] = None, - after_call: Optional[AfterCallAsync] = None, - tool_calls_list: Optional[List[ToolCall]] = None, - ) -> List[ToolResult]: + before_call: BeforeCallAsync | None = None, + after_call: AfterCallAsync | None = None, + tool_calls_list: list[ToolCall] | None = None, + ) -> list[ToolResult]: """Execute tool calls using this response's tools. By default executes ``await self.tool_calls()``; pass @@ -2200,19 +2197,19 @@ async def execute_tool_calls( await inst.prepare_async() inst._async_prepared = True - indexed_results: List[tuple[int, ToolResult]] = [] - async_tasks: List[asyncio.Task] = [] - async_task_indexes: List[int] = [] + indexed_results: list[tuple[int, ToolResult]] = [] + async_tasks: list[asyncio.Task] = [] + async_task_indexes: list[int] = [] # Defined failure semantics: a pause or error in one call must not # orphan concurrently-running siblings. Pauses and hook failures # are collected here and raised only after every task that was # started has finished. - paused: List[tuple[int, PauseChain]] = [] - failures: List[tuple[int, BaseException]] = [] + paused: list[tuple[int, PauseChain]] = [] + failures: list[tuple[int, BaseException]] = [] for idx, tc in enumerate(tool_calls_list): - tool: Optional[Tool] = tools_by_name.get(tc.name) - exception: Optional[Exception] = None + tool: Tool | None = tools_by_name.get(tc.name) + exception: Exception | None = None if tool is None or not tool.implementation: # Mirror the sync executor: append an error ToolResult so @@ -2237,11 +2234,11 @@ async def execute_tool_calls( ) ) continue - except Exception as ex: + except Exception as ex: # noqa: BLE001 failures.append((idx, ex)) break reason = "does not exist" if tool is None else "has no implementation" - msg = 'tool "{}" {}'.format(tc.name, reason) + msg = f'tool "{tc.name}" {reason}' indexed_results.append( ( idx, @@ -2292,7 +2289,7 @@ async def run_async(tc=tc, tool=tool, idx=idx): # the gather so siblings finish first. ex.tool_call = tc raise - except Exception as ex: + except Exception as ex: # noqa: BLE001 output = f"Error: {ex}" exception = ex @@ -2336,7 +2333,7 @@ async def run_async(tc=tc, tool=tool, idx=idx): ) ) continue - except Exception as ex: + except Exception as ex: # noqa: BLE001 failures.append((idx, ex)) break @@ -2360,7 +2357,7 @@ async def run_async(tc=tc, tool=tool, idx=idx): ex.tool_call = tc paused.append((idx, ex)) break - except Exception as ex: + except Exception as ex: # noqa: BLE001 output = f"Error: {ex}" exception = ex @@ -2378,7 +2375,7 @@ async def run_async(tc=tc, tool=tool, idx=idx): cb2 = after_call(tool, tc, tr) if inspect.isawaitable(cb2): await cb2 - except Exception as ex: + except Exception as ex: # noqa: BLE001 failures.append((idx, ex)) break @@ -2493,7 +2490,7 @@ async def astream_events(self): finally: pass - async def messages(self) -> List[Any]: + async def messages(self) -> list[Any]: """List of Message objects produced by this response. Awaits ``self._force()`` so ``await response.messages()`` is @@ -2523,17 +2520,17 @@ async def text(self) -> str: await self._force() return "".join(self._chunks) - async def tool_calls(self) -> List[ToolCall]: + async def tool_calls(self) -> list[ToolCall]: "Return the list of tool calls made during this response." await self._force() return self._tool_calls - def tool_calls_or_raise(self) -> List[ToolCall]: + def tool_calls_or_raise(self) -> list[ToolCall]: if not self._done: raise ValueError("Response not yet awaited") return self._tool_calls - async def json(self) -> Optional[Dict[str, Any]]: + async def json(self) -> dict[str, Any] | None: "Return the raw JSON response from the model, if available." await self._force() return self.response_json @@ -2603,7 +2600,7 @@ def fake( cls, model: "AsyncModel", prompt: str, - *attachments: List[Attachment], + *attachments: list[Attachment], system: str, response: str, ): @@ -2626,10 +2623,10 @@ def __repr__(self): text = "... not yet awaited ..." if self._done: text = "".join(self._chunks) - return "".format(self.prompt.prompt, text) + return f"" -def _append_tool_results_to_chain(chain, tool_results, attachments) -> List[Any]: +def _append_tool_results_to_chain(chain, tool_results, attachments) -> list[Any]: """Append a tool-role message carrying ToolResults to a message chain, plus a trailing user-role message for any attachments the tools returned (mimics the legacy attachments=[] kwarg behavior).""" @@ -2663,7 +2660,7 @@ def _append_tool_results_to_chain(chain, tool_results, attachments) -> List[Any] return chain -def _chain_for_tool_results(prior_response, tool_results, attachments) -> List[Any]: +def _chain_for_tool_results(prior_response, tool_results, attachments) -> list[Any]: """Build the message chain for a tool-result turn in a chain loop. Takes the prior response's full input chain + its structured @@ -2675,13 +2672,13 @@ def _chain_for_tool_results(prior_response, tool_results, attachments) -> List[A including any reasoning signatures or thoughtSignatures from the prior turn. """ - chain: List[Any] = list(prior_response.prompt.messages) + list( + chain: list[Any] = list(prior_response.prompt.messages) + list( prior_response._messages_now() ) return _append_tool_results_to_chain(chain, tool_results, attachments) -def _trailing_pending_tool_calls(messages) -> List[ToolCall]: +def _trailing_pending_tool_calls(messages) -> list[ToolCall]: """Find unresolved tool calls at the end of a message history. Returns ToolCall objects from the last assistant message containing @@ -2698,7 +2695,7 @@ def _trailing_pending_tool_calls(messages) -> List[ToolCall]: from .parts import ToolCallPart, ToolResultPart last_index = None - call_parts: List[Any] = [] + call_parts: list[Any] = [] for i, msg in enumerate(messages or []): parts = getattr(msg, "parts", None) or [] calls = [ @@ -2710,7 +2707,7 @@ def _trailing_pending_tool_calls(messages) -> List[ToolCall]: if last_index is None: return [] - results: List[Any] = [] + results: list[Any] = [] for msg in messages[last_index + 1 :]: role = getattr(msg, "role", None) if role == "tool": @@ -2747,7 +2744,7 @@ class _BaseChainResponse: prompt: "Prompt" stream: bool conversation: Optional["_BaseConversation"] = None - _key: Optional[str] = None + _key: str | None = None def __init__( self, @@ -2755,16 +2752,16 @@ def __init__( model: "_BaseModel", stream: bool, conversation: _BaseConversation, - key: Optional[str] = None, - chain_limit: Optional[int] = 10, - before_call: Optional[Union[BeforeCallSync, BeforeCallAsync]] = None, - after_call: Optional[Union[AfterCallSync, AfterCallAsync]] = None, + key: str | None = None, + chain_limit: int | None = 10, + before_call: BeforeCallSync | BeforeCallAsync | None = None, + after_call: AfterCallSync | AfterCallAsync | None = None, ): self.prompt = prompt self.model = model self.stream = stream self._key = key - self._responses: List[Any] = [] + self._responses: list[Any] = [] self.conversation = conversation self.chain_limit = chain_limit self.before_call = before_call @@ -2780,7 +2777,7 @@ def log_to_db(self, db): assert False, "Should have been a Response or AsyncResponse" sync_response.log_to_db(db) - def _pending_tool_calls(self) -> List[ToolCall]: + def _pending_tool_calls(self) -> list[ToolCall]: """Unresolved tool calls at the end of this chain's history. Non-empty when the supplied messages= end in an assistant @@ -2791,7 +2788,7 @@ def _pending_tool_calls(self) -> List[ToolCall]: return [] return _trailing_pending_tool_calls(self.prompt.messages) - def _resume_prompt(self, tool_results: List[ToolResult]) -> Prompt: + def _resume_prompt(self, tool_results: list[ToolResult]) -> Prompt: """The first prompt for a resumed chain: the original history plus a tool-role message carrying the freshly-executed results - the same shape as the chain loop's own tool-result turns.""" @@ -2817,9 +2814,9 @@ def _resume_prompt(self, tool_results: List[ToolResult]) -> Prompt: class ChainResponse(_BaseChainResponse): - _responses: List["Response"] - before_call: Optional[BeforeCallSync] = None - after_call: Optional[AfterCallSync] = None + _responses: list["Response"] + before_call: BeforeCallSync | None = None + after_call: AfterCallSync | None = None def responses(self) -> Iterator[Response]: prompt = self.prompt @@ -2850,7 +2847,7 @@ def responses(self) -> Iterator[Response]: key=self._key, conversation=self.conversation, ) - current_response: Optional[Response] = initial_response + current_response: Response | None = initial_response while current_response: count += 1 yield current_response @@ -2914,9 +2911,9 @@ def text(self) -> str: class AsyncChainResponse(_BaseChainResponse): - _responses: List["AsyncResponse"] - before_call: Optional[BeforeCallAsync] = None - after_call: Optional[AfterCallAsync] = None + _responses: list["AsyncResponse"] + before_call: BeforeCallAsync | None = None + after_call: AfterCallAsync | None = None async def responses(self) -> AsyncIterator[AsyncResponse]: prompt = self.prompt @@ -2945,7 +2942,7 @@ async def responses(self) -> AsyncIterator[AsyncResponse]: key=self._key, conversation=self.conversation, ) - current_response: Optional[AsyncResponse] = initial_response + current_response: AsyncResponse | None = initial_response while current_response: count += 1 yield current_response @@ -3018,11 +3015,11 @@ class Options(BaseModel): class _get_key_mixin: - needs_key: Optional[str] = None - key: Optional[str] = None - key_env_var: Optional[str] = None + needs_key: str | None = None + key: str | None = None + key_env_var: str | None = None - def get_key(self, explicit_key: Optional[str] = None) -> Optional[str]: + def get_key(self, explicit_key: str | None = None) -> str | None: from llm import get_key if self.needs_key is None: @@ -3043,18 +3040,16 @@ def get_key(self, explicit_key: Optional[str] = None) -> Optional[str]: return key_value # Show a useful error message - message = "No key found - add one using 'llm keys set {}'".format( - self.needs_key - ) + message = f"No key found - add one using 'llm keys set {self.needs_key}'" if self.key_env_var: - message += " or set the {} environment variable".format(self.key_env_var) + message += f" or set the {self.key_env_var} environment variable" raise NeedsKeyException(message) class _BaseModel(ABC, _get_key_mixin): model_id: str can_stream: bool = False - attachment_types: Set = set() + attachment_types: set[str] | frozenset[str] = frozenset() supports_schema = False supports_tools = False @@ -3063,7 +3058,7 @@ class Options(_Options): pass def _validate_attachments( - self, attachments: Optional[List[Attachment]] = None + self, attachments: list[Attachment] | None = None ) -> None: if attachments and not self.attachment_types: raise ValueError("This model does not support attachments") @@ -3083,16 +3078,16 @@ def __str__(self) -> str: ) def __repr__(self) -> str: - return f"<{str(self)}>" + return f"<{self!s}>" class _Model(_BaseModel): def conversation( self, - tools: Optional[List[ToolDef]] = None, - before_call: Optional[BeforeCallSync] = None, - after_call: Optional[AfterCallSync] = None, - chain_limit: Optional[int] = None, + tools: list[ToolDef] | None = None, + before_call: BeforeCallSync | None = None, + after_call: AfterCallSync | None = None, + chain_limit: int | None = None, ) -> Conversation: return Conversation( model=self, @@ -3104,18 +3099,18 @@ def conversation( def prompt( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[Union[str, Fragment]]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - system_fragments: Optional[List[Union[str, Fragment]]] = None, - messages: Optional[List[Any]] = None, + fragments: list[str | Fragment] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + system_fragments: list[str | Fragment] | None = None, + messages: list[Any] | None = None, stream: bool = True, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - options: Optional[dict] = None, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + options: dict | None = None, hide_reasoning: bool = False, **kwargs, ) -> Response: @@ -3144,21 +3139,21 @@ def prompt( def chain( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[str]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - system_fragments: Optional[List[str]] = None, - messages: Optional[List[Any]] = None, + fragments: list[str] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + system_fragments: list[str] | None = None, + messages: list[Any] | None = None, stream: bool = True, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - before_call: Optional[BeforeCallSync] = None, - after_call: Optional[AfterCallSync] = None, - key: Optional[str] = None, - options: Optional[dict] = None, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + before_call: BeforeCallSync | None = None, + after_call: AfterCallSync | None = None, + key: str | None = None, + options: dict | None = None, hide_reasoning: bool = False, ) -> ChainResponse: return self.conversation().chain( @@ -3187,7 +3182,7 @@ def execute( prompt: Prompt, stream: bool, response: Response, - conversation: Optional[Conversation], + conversation: Conversation | None, ) -> Iterator[Union[str, "StreamEvent"]]: pass @@ -3199,8 +3194,8 @@ def execute( prompt: Prompt, stream: bool, response: Response, - conversation: Optional[Conversation], - key: Optional[str], + conversation: Conversation | None, + key: str | None, ) -> Iterator[Union[str, "StreamEvent"]]: pass @@ -3208,10 +3203,10 @@ def execute( class _AsyncModel(_BaseModel): def conversation( self, - tools: Optional[List[ToolDef]] = None, - before_call: Optional[BeforeCallAsync] = None, - after_call: Optional[AfterCallAsync] = None, - chain_limit: Optional[int] = None, + tools: list[ToolDef] | None = None, + before_call: BeforeCallAsync | None = None, + after_call: AfterCallAsync | None = None, + chain_limit: int | None = None, ) -> AsyncConversation: return AsyncConversation( model=self, @@ -3223,18 +3218,18 @@ def conversation( def prompt( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[Union[str, Fragment]]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - system_fragments: Optional[List[Union[str, Fragment]]] = None, - messages: Optional[List[Any]] = None, + fragments: list[str | Fragment] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + system_fragments: list[str | Fragment] | None = None, + messages: list[Any] | None = None, stream: bool = True, - options: Optional[dict] = None, + options: dict | None = None, hide_reasoning: bool = False, **kwargs, ) -> AsyncResponse: @@ -3263,21 +3258,21 @@ def prompt( def chain( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[str]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - system_fragments: Optional[List[str]] = None, - messages: Optional[List[Any]] = None, + fragments: list[str] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + system_fragments: list[str] | None = None, + messages: list[Any] | None = None, stream: bool = True, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - before_call: Optional[BeforeCallAsync] = None, - after_call: Optional[AfterCallAsync] = None, - key: Optional[str] = None, - options: Optional[dict] = None, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + before_call: BeforeCallAsync | None = None, + after_call: AfterCallAsync | None = None, + key: str | None = None, + options: dict | None = None, hide_reasoning: bool = False, ) -> AsyncChainResponse: return self.conversation().chain( @@ -3306,11 +3301,10 @@ async def execute( prompt: Prompt, stream: bool, response: AsyncResponse, - conversation: Optional[AsyncConversation], + conversation: AsyncConversation | None, ) -> AsyncGenerator[Union[str, "StreamEvent"], None]: if False: # Ensure it's a generator type yield "" - pass class AsyncKeyModel(_AsyncModel): @@ -3320,24 +3314,23 @@ async def execute( prompt: Prompt, stream: bool, response: AsyncResponse, - conversation: Optional[AsyncConversation], - key: Optional[str], + conversation: AsyncConversation | None, + key: str | None, ) -> AsyncGenerator[Union[str, "StreamEvent"], None]: if False: # Ensure it's a generator type yield "" - pass class EmbeddingModel(ABC, _get_key_mixin): model_id: str - key: Optional[str] = None - needs_key: Optional[str] = None - key_env_var: Optional[str] = None + key: str | None = None + needs_key: str | None = None + key_env_var: str | None = None supports_text: bool = True supports_binary: bool = False - batch_size: Optional[int] = None + batch_size: int | None = None - def _check(self, item: Union[str, bytes]): + def _check(self, item: str | bytes): if not self.supports_binary and isinstance(item, bytes): raise ValueError( "This model does not support binary data, only text strings" @@ -3347,14 +3340,14 @@ def _check(self, item: Union[str, bytes]): "This model does not support text strings, only binary data" ) - def embed(self, item: Union[str, bytes]) -> List[float]: + def embed(self, item: str | bytes) -> list[float]: "Embed a single text string or binary blob, return a list of floats" self._check(item) return next(iter(self.embed_batch([item]))) def embed_multi( - self, items: Iterable[Union[str, bytes]], batch_size: Optional[int] = None - ) -> Iterator[List[float]]: + self, items: Iterable[str | bytes], batch_size: int | None = None + ) -> Iterator[list[float]]: "Embed multiple items in batches according to the model batch_size" iter_items = iter(items) effective_batch_size = self.batch_size if batch_size is None else batch_size @@ -3376,17 +3369,16 @@ def checking_iter(inner_items): yield from self.embed_batch(batch_items) @abstractmethod - def embed_batch(self, items: Iterable[Union[str, bytes]]) -> Iterator[List[float]]: + def embed_batch(self, items: Iterable[str | bytes]) -> Iterator[list[float]]: """ Embed a batch of strings or blobs, return a list of lists of floats """ - pass def __str__(self) -> str: - return "{}: {}".format(self.__class__.__name__, self.model_id) + return f"{self.__class__.__name__}: {self.model_id}" def __repr__(self) -> str: - return f"<{str(self)}>" + return f"<{self!s}>" @dataclass @@ -3395,11 +3387,11 @@ class ModelWithAliases: model: Model async_model: AsyncModel - aliases: Set[str] + aliases: set[str] def matches(self, query: str) -> bool: query_lower = query.lower() - all_strings: List[str] = [] + all_strings: list[str] = [] all_strings.extend(self.aliases) if self.model: all_strings.append(str(self.model)) @@ -3411,11 +3403,11 @@ def matches(self, query: str) -> bool: @dataclass class EmbeddingModelWithAliases: model: EmbeddingModel - aliases: Set[str] + aliases: set[str] def matches(self, query: str) -> bool: query_lower = query.lower() - all_strings: List[str] = [] + all_strings: list[str] = [] all_strings.extend(self.aliases) all_strings.append(str(self.model)) return any(query_lower in alias.lower() for alias in all_strings) diff --git a/llm/parts.py b/llm/parts.py index 5f5fd16eb..c7cff8c2b 100644 --- a/llm/parts.py +++ b/llm/parts.py @@ -13,7 +13,7 @@ import base64 from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import Any from .models import Attachment from .serialization import ( @@ -29,7 +29,7 @@ def _attachment_to_dict(att: Attachment) -> AttachmentDict: - d: Dict[str, Any] = {} + d: dict[str, Any] = {} if att.type: d["type"] = att.type if att.url: @@ -43,7 +43,7 @@ def _attachment_to_dict(att: Attachment) -> AttachmentDict: def _attachment_from_dict(d: AttachmentDict) -> Attachment: raw_content = d.get("content") - content_bytes: Optional[bytes] = None + content_bytes: bytes | None = None if isinstance(raw_content, str): content_bytes = base64.b64decode(raw_content) return Attachment( @@ -107,10 +107,10 @@ def from_dict(d: PartDict) -> "Part": @dataclass class TextPart(Part): text: str = "" - provider_metadata: Optional[Dict[str, Any]] = None + provider_metadata: dict[str, Any] | None = None def to_dict(self) -> TextPartDict: - d: Dict[str, Any] = {"type": "text", "text": self.text} + d: dict[str, Any] = {"type": "text", "text": self.text} if self.provider_metadata: d["provider_metadata"] = self.provider_metadata return d # type: ignore[return-value] @@ -130,10 +130,10 @@ class ReasoningPart(Part): text: str = "" redacted: bool = False - provider_metadata: Optional[Dict[str, Any]] = None + provider_metadata: dict[str, Any] | None = None def to_dict(self) -> ReasoningPartDict: - d: Dict[str, Any] = {"type": "reasoning", "text": self.text} + d: dict[str, Any] = {"type": "reasoning", "text": self.text} if self.redacted: d["redacted"] = True if self.provider_metadata: @@ -151,13 +151,13 @@ class ToolCallPart(Part): """ name: str = "" - arguments: Dict[str, Any] = field(default_factory=dict) - tool_call_id: Optional[str] = None + arguments: dict[str, Any] = field(default_factory=dict) + tool_call_id: str | None = None server_executed: bool = False - provider_metadata: Optional[Dict[str, Any]] = None + provider_metadata: dict[str, Any] | None = None def to_dict(self) -> ToolCallPartDict: - d: Dict[str, Any] = { + d: dict[str, Any] = { "type": "tool_call", "name": self.name, "arguments": self.arguments, @@ -177,14 +177,14 @@ class ToolResultPart(Part): name: str = "" output: str = "" - tool_call_id: Optional[str] = None + tool_call_id: str | None = None server_executed: bool = False - attachments: List[Any] = field(default_factory=list) - exception: Optional[str] = None - provider_metadata: Optional[Dict[str, Any]] = None + attachments: list[Any] = field(default_factory=list) + exception: str | None = None + provider_metadata: dict[str, Any] | None = None def to_dict(self) -> ToolResultPartDict: - d: Dict[str, Any] = { + d: dict[str, Any] = { "type": "tool_result", "name": self.name, "output": self.output, @@ -206,11 +206,11 @@ def to_dict(self) -> ToolResultPartDict: class AttachmentPart(Part): """An inline attachment (image, audio, file).""" - attachment: Optional[Attachment] = None - provider_metadata: Optional[Dict[str, Any]] = None + attachment: Attachment | None = None + provider_metadata: dict[str, Any] | None = None def to_dict(self) -> AttachmentPartDict: - d: Dict[str, Any] = {"type": "attachment"} + d: dict[str, Any] = {"type": "attachment"} if self.attachment: d["attachment"] = _attachment_to_dict(self.attachment) if self.provider_metadata: @@ -229,11 +229,11 @@ class Message: """ role: str - parts: List[Part] = field(default_factory=list) - provider_metadata: Optional[Dict[str, Any]] = None + parts: list[Part] = field(default_factory=list) + provider_metadata: dict[str, Any] | None = None def to_dict(self) -> MessageDict: - d: Dict[str, Any] = { + d: dict[str, Any] = { "role": self.role, "parts": [p.to_dict() for p in self.parts], } @@ -250,13 +250,13 @@ def from_dict(d: MessageDict) -> "Message": ) -def normalize_parts(items: Any) -> List[Part]: +def normalize_parts(items: Any) -> list[Part]: """Normalize helper inputs to a list of Part objects. Accepts str (→ TextPart), Attachment (→ AttachmentPart), Part (passed through), or a list/tuple of those (flattened one level). """ - out: List[Part] = [] + out: list[Part] = [] for item in items: if isinstance(item, Part): out.append(item) @@ -271,7 +271,7 @@ def normalize_parts(items: Any) -> List[Part]: return out -def system(*items: Any, provider_metadata: Optional[Dict[str, Any]] = None) -> Message: +def system(*items: Any, provider_metadata: dict[str, Any] | None = None) -> Message: "Build a Message with role='system'." return Message( role="system", @@ -280,7 +280,7 @@ def system(*items: Any, provider_metadata: Optional[Dict[str, Any]] = None) -> M ) -def user(*items: Any, provider_metadata: Optional[Dict[str, Any]] = None) -> Message: +def user(*items: Any, provider_metadata: dict[str, Any] | None = None) -> Message: "Build a Message with role='user'." return Message( role="user", @@ -289,9 +289,7 @@ def user(*items: Any, provider_metadata: Optional[Dict[str, Any]] = None) -> Mes ) -def assistant( - *items: Any, provider_metadata: Optional[Dict[str, Any]] = None -) -> Message: +def assistant(*items: Any, provider_metadata: dict[str, Any] | None = None) -> Message: "Build a Message with role='assistant'." return Message( role="assistant", @@ -301,7 +299,7 @@ def assistant( def tool_message( - *items: Any, provider_metadata: Optional[Dict[str, Any]] = None + *items: Any, provider_metadata: dict[str, Any] | None = None ) -> Message: "Build a Message with role='tool' (typically wrapping ToolResultParts)." return Message( @@ -343,10 +341,10 @@ class StreamEvent: type: str # "text" / "reasoning" / "tool_call_name" / # "tool_call_args" / "tool_result" chunk: str - part_index: Optional[int] = None - tool_call_id: Optional[str] = None + part_index: int | None = None + tool_call_id: str | None = None server_executed: bool = False - tool_name: Optional[str] = None + tool_name: str | None = None redacted: bool = False - provider_metadata: Optional[Dict[str, Any]] = None + provider_metadata: dict[str, Any] | None = None message_index: int = 0 diff --git a/llm/plugins.py b/llm/plugins.py index 0125ede04..49a134dac 100644 --- a/llm/plugins.py +++ b/llm/plugins.py @@ -1,8 +1,10 @@ import importlib -from importlib import metadata import os -import pluggy import sys +from importlib import metadata + +import pluggy + from . import hookspecs DEFAULT_PLUGINS = ( diff --git a/llm/serialization.py b/llm/serialization.py index 33d6400a9..b2e5ae89c 100644 --- a/llm/serialization.py +++ b/llm/serialization.py @@ -32,7 +32,7 @@ def save_messages(conn, messages: list[MessageDict]) -> None: always be present. """ -from typing import Any, Dict, List, Literal, Union +from typing import Any, Literal # NotRequired moved to typing in 3.11; use typing_extensions for 3.10 # support. typing_extensions is a transitive dep via pydantic. @@ -75,7 +75,7 @@ class AttachmentDict(TypedDict, total=False): class TextPartDict(TypedDict): type: Literal["text"] text: str - provider_metadata: NotRequired[Dict[str, Any]] + provider_metadata: NotRequired[dict[str, Any]] class ReasoningPartDict(TypedDict): @@ -85,19 +85,19 @@ class ReasoningPartDict(TypedDict): # reasoning (OpenAI GPT-5, Gemini without thoughts). The token # total lives on response usage, not on the Part. redacted: NotRequired[bool] - provider_metadata: NotRequired[Dict[str, Any]] + provider_metadata: NotRequired[dict[str, Any]] class ToolCallPartDict(TypedDict): type: Literal["tool_call"] name: str - arguments: Dict[str, Any] + arguments: dict[str, Any] tool_call_id: NotRequired[str] # True for provider-executed calls (Anthropic web search, Gemini code # execution). Adapters use this to restore provider-side blocks on # the next turn. server_executed: NotRequired[bool] - provider_metadata: NotRequired[Dict[str, Any]] + provider_metadata: NotRequired[dict[str, Any]] class ToolResultPartDict(TypedDict): @@ -107,23 +107,23 @@ class ToolResultPartDict(TypedDict): tool_call_id: NotRequired[str] server_executed: NotRequired[bool] exception: NotRequired[str] - attachments: NotRequired[List[AttachmentDict]] - provider_metadata: NotRequired[Dict[str, Any]] + attachments: NotRequired[list[AttachmentDict]] + provider_metadata: NotRequired[dict[str, Any]] class AttachmentPartDict(TypedDict): type: Literal["attachment"] attachment: NotRequired[AttachmentDict] - provider_metadata: NotRequired[Dict[str, Any]] + provider_metadata: NotRequired[dict[str, Any]] -PartDict = Union[ - TextPartDict, - ReasoningPartDict, - ToolCallPartDict, - ToolResultPartDict, - AttachmentPartDict, -] +PartDict = ( + TextPartDict + | ReasoningPartDict + | ToolCallPartDict + | ToolResultPartDict + | AttachmentPartDict +) """Discriminated union of Part dict shapes. Use with ``pydantic.TypeAdapter(PartDict)`` to validate / dispatch by ``type``. """ @@ -140,8 +140,8 @@ class MessageDict(TypedDict): """ role: str - parts: List[PartDict] - provider_metadata: NotRequired[Dict[str, Any]] + parts: list[PartDict] + provider_metadata: NotRequired[dict[str, Any]] # ---- Response + nested shapes ----------------------------------------------- @@ -152,8 +152,8 @@ class PromptDict(TypedDict): full input chain that was sent for this turn plus any options that apply.""" - messages: List[MessageDict] - options: NotRequired[Dict[str, Any]] + messages: list[MessageDict] + options: NotRequired[dict[str, Any]] system: NotRequired[str] @@ -163,7 +163,7 @@ class UsageDict(TypedDict, total=False): input: int output: int - details: Dict[str, Any] + details: dict[str, Any] class ResponseDict(TypedDict): @@ -174,7 +174,7 @@ class ResponseDict(TypedDict): model: str prompt: PromptDict - messages: List[MessageDict] + messages: list[MessageDict] # Audit fields — present on a freshly-serialized response, optional # on hand-constructed ones. id: NotRequired[str] diff --git a/llm/templates.py b/llm/templates.py index ac1b7c716..1919e0484 100644 --- a/llm/templates.py +++ b/llm/templates.py @@ -1,6 +1,7 @@ -from pydantic import BaseModel, ConfigDict import string -from typing import Optional, Any, Dict, List, Tuple +from typing import Any + +from pydantic import BaseModel, ConfigDict class AttachmentType(BaseModel): @@ -12,20 +13,20 @@ class Template(BaseModel): """A reusable prompt template.""" name: str - prompt: Optional[str] = None - system: Optional[str] = None - attachments: Optional[List[str]] = None - attachment_types: Optional[List[AttachmentType]] = None - model: Optional[str] = None - defaults: Optional[Dict[str, Any]] = None - options: Optional[Dict[str, Any]] = None - extract: Optional[bool] = None # For extracting fenced code blocks - extract_last: Optional[bool] = None - schema_object: Optional[dict] = None - fragments: Optional[List[str]] = None - system_fragments: Optional[List[str]] = None - tools: Optional[List[str]] = None - functions: Optional[str] = None + prompt: str | None = None + system: str | None = None + attachments: list[str] | None = None + attachment_types: list[AttachmentType] | None = None + model: str | None = None + defaults: dict[str, Any] | None = None + options: dict[str, Any] | None = None + extract: bool | None = None # For extracting fenced code blocks + extract_last: bool | None = None + schema_object: dict | None = None + fragments: list[str] | None = None + system_fragments: list[str] | None = None + tools: list[str] | None = None + functions: str | None = None model_config = ConfigDict(extra="forbid") @@ -39,8 +40,8 @@ def __init__(self, **data): self._functions_is_trusted = False def evaluate( - self, input: str, params: Optional[Dict[str, Any]] = None - ) -> Tuple[Optional[str], Optional[str]]: + self, input: str, params: dict[str, Any] | None = None + ) -> tuple[str | None, str | None]: """Evaluate the template with the given input and parameters, returning (prompt, system).""" params = params or {} params["input"] = input @@ -48,8 +49,8 @@ def evaluate( for k, v in self.defaults.items(): if k not in params: params[k] = v - prompt: Optional[str] = None - system: Optional[str] = None + prompt: str | None = None + system: str | None = None if not self.prompt: system = self.interpolate(self.system, params) prompt = input @@ -68,7 +69,7 @@ def vars(self) -> set: return all_vars @classmethod - def interpolate(cls, text: Optional[str], params: Dict[str, Any]) -> Optional[str]: + def interpolate(cls, text: str | None, params: dict[str, Any]) -> str | None: """Substitute template variables in text with values from params, raising MissingVariables if any are absent.""" if not text: return text @@ -83,7 +84,7 @@ def interpolate(cls, text: Optional[str], params: Dict[str, Any]) -> Optional[st return string_template.substitute(**params) @staticmethod - def extract_vars(string_template: string.Template) -> List[str]: + def extract_vars(string_template: string.Template) -> list[str]: """Extract and return the list of named variable identifiers from a string.Template.""" return [ match.group("named") diff --git a/llm/tools.py b/llm/tools.py index 5ac0a7dcb..0970205c7 100644 --- a/llm/tools.py +++ b/llm/tools.py @@ -1,6 +1,6 @@ +import time from datetime import datetime, timezone from importlib.metadata import version -import time def llm_version() -> str: @@ -12,7 +12,7 @@ def llm_time() -> dict: "Returns the current time, as local time and UTC" # Get current times utc_time = datetime.now(timezone.utc) - local_time = datetime.now() + local_time = datetime.now(timezone.utc).astimezone() # Get timezone information local_tz_name = time.tzname[time.localtime().tm_isdst] diff --git a/llm/utils.py b/llm/utils.py index 587f19284..86920837f 100644 --- a/llm/utils.py +++ b/llm/utils.py @@ -1,19 +1,18 @@ -import click import hashlib -import httpx import itertools import json +import os import pathlib -import puremagic import re -import sqlite_utils import textwrap -from typing import Any, List, Dict, Optional, Tuple, Type -import os import threading import time -from typing import Final +from typing import Any, Final +import click +import httpx +import puremagic +import sqlite_utils from ulid import ULID MIME_TYPE_FIXES = { @@ -34,7 +33,7 @@ def id(self): return hashlib.sha256(self.encode("utf-8")).hexdigest() -def mimetype_from_string(content) -> Optional[str]: +def mimetype_from_string(content) -> str | None: try: type_ = puremagic.from_string(content, mime=True) return MIME_TYPE_FIXES.get(type_, type_) @@ -42,7 +41,7 @@ def mimetype_from_string(content) -> Optional[str]: return None -def mimetype_from_path(path) -> Optional[str]: +def mimetype_from_path(path) -> str | None: try: type_ = puremagic.from_file(path, mime=True) return MIME_TYPE_FIXES.get(type_, type_) @@ -51,8 +50,8 @@ def mimetype_from_path(path) -> Optional[str]: def dicts_to_table_string( - headings: List[str], dicts: List[Dict[str, str]] -) -> List[str]: + headings: list[str], dicts: list[dict[str, str]] +) -> list[str]: max_lengths = [len(h) for h in headings] # Compute maximum length for each column @@ -179,7 +178,7 @@ def token_usage_string(input_tokens, output_tokens, token_details) -> str: return ", ".join(bits) -def extract_fenced_code_block(text: str, last: bool = False) -> Optional[str]: +def extract_fenced_code_block(text: str, last: bool = False) -> str | None: """ Extracts and returns Markdown fenced code block found in the given text. @@ -217,7 +216,7 @@ def extract_fenced_code_block(text: str, last: bool = False) -> Optional[str]: return None -def make_schema_id(schema: dict) -> Tuple[str, str]: +def make_schema_id(schema: dict) -> tuple[str, str]: schema_json = json.dumps(schema, separators=(",", ":")) schema_id = hashlib.blake2b(schema_json.encode(), digest_size=16).hexdigest() return schema_id, schema_json @@ -282,9 +281,9 @@ def resolve_schema_input(db, schema_input, load_template): template = load_template(name) schema_object = template.schema_object except ValueError: - raise click.ClickException("Invalid template: {}".format(name)) + raise click.ClickException(f"Invalid template: {name}") if not schema_object: - raise click.ClickException("Template '{}' has no schema".format(name)) + raise click.ClickException(f"Template '{name}' has no schema") return template.schema_object if schema_input.strip().startswith("{"): try: @@ -351,7 +350,7 @@ def schema_summary(schema: dict) -> str: return "" -def schema_dsl(schema_dsl: str, multi: bool = False) -> Dict[str, Any]: +def schema_dsl(schema_dsl: str, multi: bool = False) -> dict[str, Any]: """ Build a JSON schema from a concise schema string. @@ -372,7 +371,7 @@ def schema_dsl(schema_dsl: str, multi: bool = False) -> Dict[str, Any]: } # Initialize the schema dictionary with required elements - json_schema: Dict[str, Any] = {"type": "object", "properties": {}, "required": []} + json_schema: dict[str, Any] = {"type": "object", "properties": {}, "required": []} # Check if the schema is newline-separated or comma-separated if "\n" in schema_dsl: @@ -487,9 +486,13 @@ def ensure_fragment(db, content): source = content.source with db.conn: db.execute(sql, {"hash": hash_id, "content": content, "source": source}) - return list( - db.query("select id from fragments where hash = :hash", {"hash": hash_id}) - )[0]["id"] + return next( + iter( + db.query( + "select id from fragments where hash = :hash", {"hash": hash_id} + ) + ) + )["id"] def ensure_tool(db, tool): @@ -509,9 +512,13 @@ def ensure_tool(db, tool): "plugin": tool.plugin, }, ) - return list( - db.query("select id from tools where hash = :hash", {"hash": tool.hash()}) - )[0]["id"] + return next( + iter( + db.query( + "select id from tools where hash = :hash", {"hash": tool.hash()} + ) + ) + )["id"] def maybe_fenced_code(content: str) -> str: @@ -551,7 +558,7 @@ def has_plugin_prefix(value: str) -> bool: return bool(_plugin_prefix_re.match(value)) -def _parse_kwargs(arg_str: str) -> Dict[str, Any]: +def _parse_kwargs(arg_str: str) -> dict[str, Any]: """Parse key=value pairs where each value is valid JSON.""" tokens = [] buf = [] @@ -588,7 +595,7 @@ def _parse_kwargs(arg_str: str) -> Dict[str, Any]: if buf: tokens.append("".join(buf).strip()) - kwargs: Dict[str, Any] = {} + kwargs: dict[str, Any] = {} for token in tokens: if not token: continue @@ -605,7 +612,7 @@ def _parse_kwargs(arg_str: str) -> Dict[str, Any]: return kwargs -def instantiate_from_spec(class_map: Dict[str, Type], spec: str): +def instantiate_from_spec(class_map: dict[str, type], spec: str): """ Instantiate a class from a specification string with flexible argument formats. @@ -665,7 +672,7 @@ def instantiate_from_spec(class_map: Dict[str, Type], spec: str): return cls(**kw) # Starts with quote / number / [ / t f n for single positional JSON value - if re.match(r'\s*(["\[\d\-]|true|false|null)', arg_body, re.I): + if re.match(r'\s*(["\[\d\-]|true|false|null)', arg_body, re.IGNORECASE): try: positional_value = json.loads(arg_body) except json.JSONDecodeError as e: @@ -682,7 +689,7 @@ def instantiate_from_spec(class_map: Dict[str, Type], spec: str): RANDOMNESS_LEN = 10 _lock: Final = threading.Lock() -_last: Optional[bytes] = None # 16-byte last produced ULID +_last: bytes | None = None # 16-byte last produced ULID def monotonic_ulid() -> ULID: diff --git a/pyproject.toml b/pyproject.toml index 070773e62..36a6dc2f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ dev = [ "mypy>=1.10.0", "black>=26.3.1", "pytest-recording", - "ruff", + "ruff>=0.16.0", "syrupy", "types-click", "types-PyYAML", diff --git a/ruff.toml b/ruff.toml index 1ecd5d03a..567b4d53f 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,4 +1,2 @@ line-length = 160 - -[lint] -select = ["E4", "E7", "E9", "F"] +target-version = "py310" diff --git a/tests/conftest.py b/tests/conftest.py index f26b4e99c..a58240887 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,14 +1,15 @@ import importlib.metadata -import pytest -import sqlite_utils import json import sqlite3 -import llm + import llm_echo -from llm.plugins import pm +import pytest +import sqlite_utils from pydantic import Field from pytest_httpx import IteratorStream -from typing import Optional + +import llm +from llm.plugins import pm def pytest_configure(config): @@ -23,8 +24,8 @@ def pytest_report_header(config): conn.close() sqlite_utils_version = importlib.metadata.version("sqlite-utils") return [ - "SQLite: {}".format(version), - "sqlite-utils: {}".format(sqlite_utils_version), + f"SQLite: {version}", + f"sqlite-utils: {sqlite_utils_version}", ] @@ -63,13 +64,13 @@ def env_setup(monkeypatch, user_path): class MockModel(llm.Model): model_id = "mock" - attachment_types = {"image/png", "audio/wav"} + attachment_types = frozenset({"image/png", "audio/wav"}) can_stream = True supports_schema = True supports_tools = True class Options(llm.Options): - max_tokens: Optional[int] = Field( + max_tokens: int | None = Field( description="Maximum number of tokens to generate.", default=None ) @@ -302,7 +303,7 @@ def stream_events(): } ) ).encode("utf-8") - yield "data: [DONE]\n\n".encode("utf-8") + yield b"data: [DONE]\n\n" @pytest.fixture @@ -408,7 +409,7 @@ def stream_completion_events(): } ) ).encode("utf-8") - yield "data: [DONE]\n\n".encode("utf-8") + yield b"data: [DONE]\n\n" @pytest.fixture diff --git a/tests/test_aliases.py b/tests/test_aliases.py index 2949a8007..1645bd9e2 100644 --- a/tests/test_aliases.py +++ b/tests/test_aliases.py @@ -1,10 +1,12 @@ -from click.testing import CliRunner -from llm.cli import cli -import llm import json -import pytest import re +import pytest +from click.testing import CliRunner + +import llm +from llm.cli import cli + @pytest.mark.parametrize("model_id_or_alias", ("gpt-3.5-turbo", "chatgpt")) def test_set_alias(model_id_or_alias): @@ -30,16 +32,17 @@ def test_cli_aliases_list(args): runner = CliRunner() result = runner.invoke(cli, args) assert result.exit_code == 0 - for line in ( - "3.5 : gpt-3.5-turbo\n" - "chatgpt : gpt-3.5-turbo\n" - "chatgpt-16k : gpt-3.5-turbo-16k\n" - "3.5-16k : gpt-3.5-turbo-16k\n" - "4 : gpt-4\n" - "gpt4 : gpt-4\n" - "e-demo : embed-demo (embedding)\n" - "ada : text-embedding-ada-002 (embedding)\n" - ).split("\n"): + for line in [ + "3.5 : gpt-3.5-turbo", + "chatgpt : gpt-3.5-turbo", + "chatgpt-16k : gpt-3.5-turbo-16k", + "3.5-16k : gpt-3.5-turbo-16k", + "4 : gpt-4", + "gpt4 : gpt-4", + "e-demo : embed-demo (embedding)", + "ada : text-embedding-ada-002 (embedding)", + "", + ]: line = line.strip() if not line: continue diff --git a/tests/test_async.py b/tests/test_async.py index 30e36e3e8..5a96bfbfb 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -1,6 +1,7 @@ -import llm import pytest +import llm + @pytest.mark.asyncio async def test_async_model(async_mock_model): diff --git a/tests/test_async_parity.py b/tests/test_async_parity.py index b20fb6ecc..a84e567a6 100644 --- a/tests/test_async_parity.py +++ b/tests/test_async_parity.py @@ -7,9 +7,10 @@ import json -import llm import pytest +import llm + # ---- basic sanity: both variants are registered -------------------- @@ -91,6 +92,7 @@ async def test_async_from_row_response_messages_synthesized(tmp_path): response.messages from _chunks+_tool_calls so follow-up chains don't silently drop the assistant turn.""" import sqlite_utils + from llm.migrations import migrate model = llm.get_async_model("echo") @@ -123,6 +125,7 @@ async def test_async_load_conversation_follow_up_preserves_chain(tmp_path): load_conversation, a follow-up turn's prompt.messages is the full [user, assistant, user] chain — not missing the assistant.""" import sqlite_utils + from llm.cli import load_conversation from llm.migrations import migrate diff --git a/tests/test_attachments.py b/tests/test_attachments.py index 6e20dd7d0..88f9af3a6 100644 --- a/tests/test_attachments.py +++ b/tests/test_attachments.py @@ -1,10 +1,12 @@ -from click.testing import CliRunner import os import sys from unittest.mock import ANY + +import pytest +from click.testing import CliRunner + import llm from llm import cli -import pytest TINY_PNG = ( b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\xa6\x00\x00\x01\x1a" @@ -47,8 +49,8 @@ def test_prompt_attachment(mock_model, logs_db, attachment_type, attachment_cont conversation = conversations[0] assert conversation["model"] == "mock" assert conversation["name"] == "describe file" - response = list(logs_db["responses"].rows)[0] - attachment = list(logs_db["attachments"].rows)[0] + response = next(iter(logs_db["responses"].rows)) + attachment = next(iter(logs_db["attachments"].rows)) assert attachment == { "id": ANY, "type": attachment_type, @@ -56,7 +58,7 @@ def test_prompt_attachment(mock_model, logs_db, attachment_type, attachment_cont "url": None, "content": attachment_content, } - prompt_attachment = list(logs_db["prompt_attachments"].rows)[0] + prompt_attachment = next(iter(logs_db["prompt_attachments"].rows)) assert prompt_attachment["attachment_id"] == attachment["id"] assert prompt_attachment["response_id"] == response["id"] diff --git a/tests/test_chat.py b/tests/test_chat.py index dbd77ab4e..098161330 100644 --- a/tests/test_chat.py +++ b/tests/test_chat.py @@ -1,12 +1,14 @@ -from click.testing import CliRunner +import json import re +import sys +import textwrap from unittest.mock import ANY -import json -import llm.cli + import pytest import sqlite_utils -import sys -import textwrap +from click.testing import CliRunner + +import llm.cli @pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows") @@ -390,7 +392,7 @@ def test_chat_fragments(tmpdir): output = runner.invoke( llm.cli.cli, ["chat", "-m", "echo", "-f", path1], - input=("hi\n!fragment {}\nquit\n".format(path2)), + input=(f"hi\n!fragment {path2}\nquit\n"), ).output assert '"prompt": "one' in output assert '"prompt": "two"' in output diff --git a/tests/test_chat_templates.py b/tests/test_chat_templates.py index 7687d78d9..56404ff44 100644 --- a/tests/test_chat_templates.py +++ b/tests/test_chat_templates.py @@ -1,7 +1,9 @@ -from click.testing import CliRunner import sys -import llm.cli + import pytest +from click.testing import CliRunner + +import llm.cli @pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows") diff --git a/tests/test_cli_openai_models.py b/tests/test_cli_openai_models.py index 7da3fb442..f42cef977 100644 --- a/tests/test_cli_openai_models.py +++ b/tests/test_cli_openai_models.py @@ -1,9 +1,11 @@ -from click.testing import CliRunner import json -import llm -from llm.cli import cli + import pytest import sqlite_utils +from click.testing import CliRunner + +import llm +from llm.cli import cli @pytest.fixture diff --git a/tests/test_cli_options.py b/tests/test_cli_options.py index ce8a704b4..901ba00e3 100644 --- a/tests/test_cli_options.py +++ b/tests/test_cli_options.py @@ -1,7 +1,9 @@ +import json + +import pytest from click.testing import CliRunner + from llm.cli import cli -import pytest -import json @pytest.mark.parametrize( diff --git a/tests/test_embed.py b/tests/test_embed.py index 9b9c809cb..292170c04 100644 --- a/tests/test_embed.py +++ b/tests/test_embed.py @@ -1,9 +1,11 @@ import json -import llm -from llm.embeddings import Entry +from unittest.mock import ANY + import pytest import sqlite_utils -from unittest.mock import ANY + +import llm +from llm.embeddings import Entry def test_demo_plugin(): @@ -20,7 +22,7 @@ def test_demo_plugin(): ) def test_embed_huge_list(batch_size, expected_batches): model = llm.get_embedding_model("embed-demo") - huge_list = ("hello {}".format(i) for i in range(1000)) + huge_list = (f"hello {i}" for i in range(1000)) kwargs = {} if batch_size: kwargs["batch_size"] = batch_size @@ -120,7 +122,7 @@ def test_embed_multi(with_metadata, batch_size, expected_batches): collection = llm.Collection("test", db, model_id="embed-demo") model = collection.model() assert getattr(model, "batch_count", 0) == 0 - ids_and_texts = ((str(i), "hello {}".format(i)) for i in range(1000)) + ids_and_texts = ((str(i), f"hello {i}") for i in range(1000)) kwargs = {} if batch_size is not None: kwargs["batch_size"] = batch_size diff --git a/tests/test_embed_cli.py b/tests/test_embed_cli.py index c8e8b8ce6..11b67f56c 100644 --- a/tests/test_embed_cli.py +++ b/tests/test_embed_cli.py @@ -1,13 +1,15 @@ -from click.testing import CliRunner -from llm.cli import cli -from llm import Collection import json import pathlib -import pytest -import sqlite_utils import sys from unittest.mock import ANY +import pytest +import sqlite_utils +from click.testing import CliRunner + +from llm import Collection +from llm.cli import cli + @pytest.mark.parametrize( "format_,expected", @@ -356,7 +358,7 @@ def test_embed_multi_files_binary_store(tmpdir): assert result.exit_code == 0 db = sqlite_utils.Database(str(db_path)) assert db["embeddings"].count == 1 - row = list(db["embeddings"].rows)[0] + row = next(iter(db["embeddings"].rows)) assert row == { "collection_id": 1, "id": "file.bin", diff --git a/tests/test_encode_decode.py b/tests/test_encode_decode.py index 1310c6db2..5ff04470e 100644 --- a/tests/test_encode_decode.py +++ b/tests/test_encode_decode.py @@ -1,6 +1,7 @@ -import llm -import pytest import numpy as np +import pytest + +import llm @pytest.mark.parametrize( diff --git a/tests/test_fragments_cli.py b/tests/test_fragments_cli.py index 28fd3b4e9..d52f3179e 100644 --- a/tests/test_fragments_cli.py +++ b/tests/test_fragments_cli.py @@ -1,12 +1,14 @@ -from click.testing import CliRunner +import os +import textwrap from importlib.metadata import version -from llm.cli import cli -from llm.migrations import migrate from unittest import mock -import os -import yaml + import sqlite_utils -import textwrap +import yaml +from click.testing import CliRunner + +from llm.cli import cli +from llm.migrations import migrate def test_fragments_set_show_remove(user_path): @@ -138,7 +140,7 @@ def test_fragment_url_user_agent(mocked_openai_chat, user_path): # Verify the User-Agent header was sent for the fragment URL request requests = mocked_openai_chat.get_requests() - fragment_request = [r for r in requests if "example.com" in str(r.url)][0] + fragment_request = next(r for r in requests if "example.com" in str(r.url)) llm_version = version("llm") expected_user_agent = f"llm/{llm_version} (https://llm.datasette.io/)" assert fragment_request.headers["User-Agent"] == expected_user_agent diff --git a/tests/test_keys.py b/tests/test_keys.py index ae142d002..1cb71b430 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -1,10 +1,12 @@ -from click.testing import CliRunner import json -from llm.cli import cli import pathlib -import pytest import sys +import pytest +from click.testing import CliRunner + +from llm.cli import cli + @pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows") @pytest.mark.parametrize("env", ({}, {"LLM_USER_PATH": "/tmp/llm-keys-test"})) @@ -81,7 +83,7 @@ def test_uses_correct_key(mocked_openai_chat, monkeypatch, tmpdir): def assert_key(key): request = mocked_openai_chat.get_requests()[-1] - assert request.headers["Authorization"] == "Bearer {}".format(key) + assert request.headers["Authorization"] == f"Bearer {key}" runner = CliRunner() diff --git a/tests/test_llm.py b/tests/test_llm.py index 29262abeb..6293ca232 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -1,14 +1,16 @@ -from click.testing import CliRunner -import llm -from llm.cli import cli -from llm.models import Usage import json import os import pathlib -from pydantic import BaseModel +from unittest import mock + import pytest import sqlite_utils -from unittest import mock +from click.testing import CliRunner +from pydantic import BaseModel + +import llm +from llm.cli import cli +from llm.models import Usage def test_version(): diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index 566d5a7ea..fb5c8d020 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -1,19 +1,21 @@ -from click.testing import CliRunner -from llm.cli import cli -from llm.migrations import migrate -from llm.utils import monotonic_ulid -from llm import Fragment import datetime import json import pathlib -import pytest import re -import sqlite_utils import sys import textwrap import time -from ulid import ULID + +import pytest +import sqlite_utils import yaml +from click.testing import CliRunner +from ulid import ULID + +from llm import Fragment +from llm.cli import cli +from llm.migrations import migrate +from llm.utils import monotonic_ulid SINGLE_ID = "5843577700ba729bb14c327b30441885" MULTI_ID = "4860edd987df587d042a9eb2b299ce5c" @@ -371,7 +373,6 @@ def test_logs_filtered(user_path, model, path_option): ("llama", ["-m", "davinci"], ["doc1", "doc3"]), ("llama", ["-m", "davinci2"], []), # Adding -l/--latest should return latest first (order by id desc) - ("llama", [], ["doc1", "doc3"]), ("llama", ["-l"], ["doc3", "doc1"]), ("llama", ["--latest"], ["doc3", "doc1"]), ), @@ -1098,7 +1099,7 @@ def test_logs_resolved_model(logs_db, mock_model, async_mock_model, async_): assert result.exit_code == 0 # Should have logged the resolved model name assert logs_db["responses"].count - response = list(logs_db["responses"].rows)[0] + response = next(iter(logs_db["responses"].rows)) assert response["model"] == "mock" assert response["resolved_model"] == "resolved-mock" diff --git a/tests/test_migrate.py b/tests/test_migrate.py index 705021100..a6037a155 100644 --- a/tests/test_migrate.py +++ b/tests/test_migrate.py @@ -1,9 +1,10 @@ -import llm -from llm.migrations import migrate -from llm.embeddings_migrations import embeddings_migrations import pytest import sqlite_utils +import llm +from llm.embeddings_migrations import embeddings_migrations +from llm.migrations import migrate + EXPECTED = { "id": str, "model": str, diff --git a/tests/test_openai_messages.py b/tests/test_openai_messages.py index 9d2ae911d..65db37d46 100644 --- a/tests/test_openai_messages.py +++ b/tests/test_openai_messages.py @@ -22,7 +22,7 @@ def _sse(delta, finish_reason=None, usage=None, tool_calls=None): chunk["choices"][0]["delta"]["tool_calls"] = tool_calls if usage is not None: chunk["usage"] = usage - return f"data: {json.dumps(chunk)}\n\n".encode("utf-8") + return f"data: {json.dumps(chunk)}\n\n".encode() def _text_stream(): diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index 50b981577..ebca2a6dc 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -3,16 +3,17 @@ import json import os -import llm import pytest from pytest_httpx import IteratorStream +import llm + API_KEY = os.environ.get("PYTEST_OPENAI_API_KEY", None) or "badkey" def _responses_sse(event_type, data): data = {"type": event_type, **data} - return f"event: {event_type}\ndata: {json.dumps(data)}\n\n".encode("utf-8") + return f"event: {event_type}\ndata: {json.dumps(data)}\n\n".encode() def _responses_reasoning_summary_stream(): @@ -239,7 +240,7 @@ def test_responses_input_translation(): model = llm.get_model("gpt-5.5") class FakePrompt: - messages = [ + messages = ( Message(role="system", parts=[TextPart(text="be brief")]), Message(role="user", parts=[TextPart(text="2 + 2?")]), Message( @@ -256,7 +257,7 @@ class FakePrompt: role="tool", parts=[ToolResultPart(name="add", output="4", tool_call_id="call_abc")], ), - ] + ) items, instructions = model._build_responses_input(FakePrompt()) assert instructions == "be brief" @@ -282,11 +283,11 @@ def test_responses_input_translation_assistant_text_uses_easy_input_message(): model = llm.get_model("gpt-5.5") class FakePrompt: - messages = [ + messages = ( Message(role="user", parts=[TextPart(text="hello")]), Message(role="assistant", parts=[TextPart(text="first-ok")]), Message(role="user", parts=[TextPart(text="what next?")]), - ] + ) items, instructions = model._build_responses_input(FakePrompt()) @@ -526,7 +527,7 @@ def multiply(a: int, b: int) -> int: ) output = "".join(chain) assert "2869461" in output.replace(",", "") - first, second = chain._responses + first, _second = chain._responses assert first.tool_calls()[0].arguments == {"a": 1231, "b": 2331} diff --git a/tests/test_options_parameter.py b/tests/test_options_parameter.py index df4c09a3a..deb00f544 100644 --- a/tests/test_options_parameter.py +++ b/tests/test_options_parameter.py @@ -87,9 +87,10 @@ class AsyncModelWithOption(llm.AsyncModel): class Options(llm.Options): from typing import Optional as _Opt + from pydantic import Field as _Field - max_tokens: _Opt[int] = _Field(default=None) + max_tokens: int | None = _Field(default=None) async def execute(self, prompt, stream, response, conversation): yield "ok" diff --git a/tests/test_parts.py b/tests/test_parts.py index f2a89bc75..1f0771aa4 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -1,5 +1,7 @@ import json + import pytest + import llm @@ -446,7 +448,7 @@ def test_family_mismatch_at_same_part_index_raises(self, mock_model): response = mock_model.prompt("hi") response.text() with pytest.raises(ValueError, match="part_index"): - response.messages() # noqa: B018 + response.messages() def test_provider_metadata_merges_last_wins(self, mock_model): events = [ @@ -909,8 +911,8 @@ def test_attachments_join_user_message(self, mock_model): ] def test_tool_results_become_tool_role_message(self, mock_model): - from llm.models import Prompt from llm import ToolResult + from llm.models import Prompt tr = ToolResult(name="t", output="ok", tool_call_id="c1") p = Prompt(None, model=mock_model, tool_results=[tr]) @@ -1120,6 +1122,7 @@ def test_from_row_response_messages_synthesized_from_chunks( self, mock_model, tmp_path ): import sqlite_utils + from llm.migrations import migrate mock_model.enqueue(["answer text"]) @@ -1149,8 +1152,9 @@ def test_llm_dash_c_chain_preserves_prior_assistant_turn( """End-to-end: a follow-up turn via load_conversation must send [user(q1), assistant(a1), user(q2)] — not drop the assistant.""" import sqlite_utils - from llm.migrations import migrate + from llm.cli import load_conversation + from llm.migrations import migrate mock_model.enqueue(["first answer"]) mock_model.enqueue(["second answer"]) @@ -1179,6 +1183,7 @@ def test_llm_dash_c_after_logged_tool_chain_preserves_full_chain( assistant tool_use. Otherwise Anthropic sees an orphan tool_result at the start of the continued request.""" import sqlite_utils + from llm.cli import load_conversation from llm.migrations import migrate @@ -1704,8 +1709,7 @@ def execute(self, prompt, stream, response, conversation): yield "done" return msgs = self._queue.pop(0) - for m in msgs: - yield m + yield from msgs if not response._tool_calls: response.add_tool_call(tool_call) @@ -1731,8 +1735,7 @@ def execute(self, prompt, stream, response, conversation): yield "done" return msgs = self._queue.pop(0) - for m in msgs: - yield m + yield from msgs if not response._tool_calls: response.add_tool_call(tool_call) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 6197ef5e2..57bd17991 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,15 +1,16 @@ -from click.testing import CliRunner -import click import importlib import json -import llm -from llm.tools import llm_version, llm_time -from llm import cli, hookimpl, plugins, get_template_loaders, get_fragment_loaders import pathlib import re from unittest.mock import ANY + +import click import pytest -import textwrap +from click.testing import CliRunner + +import llm +from llm import cli, get_fragment_loaders, get_template_loaders, hookimpl, plugins +from llm.tools import llm_time, llm_version def test_register_commands(): @@ -176,16 +177,20 @@ def register_fragment_loaders(self, register): cli.cli, ["-m", "echo", "-f", "mixed:x"], catch_exceptions=False ) assert result3.exit_code == 0 - result3.output.strip == textwrap.dedent("""\ - system: - - - prompt: - one:x - - attachments: - - https://example.com/attachment.png - """).strip() + assert json.loads(result3.output) == { + "prompt": "one:x", + "system": "", + "attachments": [ + { + "type": None, + "path": None, + "url": "https://example.com/attachment.png", + "id": ANY, + } + ], + "stream": True, + "previous": [], + } finally: plugins.pm.unregister(name="FragmentLoadersPlugin") @@ -383,7 +388,7 @@ def register_tools(self, register): assert '"output": "HI"' in result4.output # Now check in the database - tool_row = [row for row in logs_db["tools"].rows][0] + tool_row = next(iter(logs_db["tools"].rows)) assert tool_row["name"] == "upper" assert tool_row["plugin"] == "ToolsPlugin" @@ -760,7 +765,7 @@ def after_call(tool, tool_call, tool_result): [ "prompt", "-T", - "Filesystem({})".format(json.dumps(str(my_dir2))), + f"Filesystem({json.dumps(str(my_dir2))})", json.dumps({"tool_calls": [{"name": "Filesystem_list_files"}]}), "-m", "echo", @@ -894,7 +899,7 @@ def test_toolbox_logging_async(logs_db, tmpdir): "-T", "Memory", "--tool", - "Filesystem({})".format(json.dumps(str(path))), + f"Filesystem({json.dumps(str(path))})", json.dumps( { "tool_calls": [ diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 3cc1c868c..2de04c444 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -7,16 +7,17 @@ """ import json + import pytest -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError import llm from llm.serialization import ( AttachmentPartDict, MessageDict, PartDict, - ResponseDict, ReasoningPartDict, + ResponseDict, TextPartDict, ToolCallPartDict, ToolResultPartDict, @@ -147,7 +148,7 @@ def test_attachment_part_validates_as_part_dict(self): TypeAdapter(PartDict).validate_python(d) def test_unknown_type_rejected(self): - with pytest.raises(Exception): + with pytest.raises(ValidationError): TypeAdapter(PartDict).validate_python({"type": "nonsense", "text": "x"}) diff --git a/tests/test_templates.py b/tests/test_templates.py index 38229619b..9ef2d8888 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -1,15 +1,17 @@ -from click.testing import CliRunner -from importlib.metadata import version import json -from llm import Template, Toolbox, hookimpl, user_dir -from llm.cli import cli -from llm.plugins import pm import os -from unittest import mock import pathlib -import pytest import textwrap +from importlib.metadata import version +from unittest import mock + +import pytest import yaml +from click.testing import CliRunner + +from llm import Template, Toolbox, hookimpl, user_dir +from llm.cli import cli +from llm.plugins import pm @pytest.mark.parametrize( diff --git a/tests/test_tools.py b/tests/test_tools.py index 81d01dc4b..e312a0d7a 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -1,16 +1,18 @@ import asyncio +import json +import os import re -from click.testing import CliRunner +import time from importlib.metadata import version -import json + +import pytest +import sqlite_utils +from click.testing import CliRunner + import llm -from llm import cli, CancelToolCall +from llm import CancelToolCall, cli from llm.migrations import migrate from llm.tools import llm_time -import os -import pytest -import sqlite_utils -import time API_KEY = os.environ.get("PYTEST_OPENAI_API_KEY", None) or "badkey" @@ -544,7 +546,6 @@ def before(tool, tool_call): if tool.name == "t1": raise CancelToolCall("skip1") # allow t2 - return None calls = [ {"name": "t1"}, @@ -583,7 +584,6 @@ async def t2() -> str: async def before(tool, tool_call): if tool.name == "t1": raise CancelToolCall("skip1") - return None calls = [ {"name": "t1"}, diff --git a/tests/test_tools_streaming.py b/tests/test_tools_streaming.py index 87a27ac65..1a64cac9f 100644 --- a/tests/test_tools_streaming.py +++ b/tests/test_tools_streaming.py @@ -1,8 +1,10 @@ -import llm -from llm.tools import llm_version import os + import pytest +import llm +from llm.tools import llm_version + API_KEY = os.environ.get("PYTEST_OPENAI_API_KEY", None) or "badkey" diff --git a/tests/test_utils.py b/tests/test_utils.py index 51fb8754f..aa9c8093c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,15 +1,17 @@ import json + import pytest + +from llm import Toolbox, get_key from llm.utils import ( extract_fenced_code_block, instantiate_from_spec, maybe_fenced_code, + monotonic_ulid, schema_dsl, simplify_usage_dict, truncate_string, - monotonic_ulid, ) -from llm import get_key, Toolbox @pytest.mark.parametrize( @@ -83,22 +85,28 @@ def test_simplify_usage_dict(input_data, expected_output): None, ], [ - "First code block:\n\n```python\ndef foo():\n return 'bar'\n```\n\n" - "Second code block:\n\n```javascript\nfunction foo() {\n return 'bar';\n}\n```", + ( + "First code block:\n\n```python\ndef foo():\n return 'bar'\n```\n\n" + "Second code block:\n\n```javascript\nfunction foo() {\n return 'bar';\n}\n```" + ), False, "def foo():\n return 'bar'\n", ], [ - "First code block:\n\n```python\ndef foo():\n return 'bar'\n```\n\n" - "Second code block:\n\n```javascript\nfunction foo() {\n return 'bar';\n}\n```", + ( + "First code block:\n\n```python\ndef foo():\n return 'bar'\n```\n\n" + "Second code block:\n\n```javascript\nfunction foo() {\n return 'bar';\n}\n```" + ), True, "function foo() {\n return 'bar';\n}\n", ], [ - "First code block:\n\n```python\ndef foo():\n return 'bar'\n```\n\n" - # This one has trailing whitespace after the second code block: - # https://github.com/simonw/llm/pull/718#issuecomment-2613177036 - "Second code block:\n\n```javascript\nfunction foo() {\n return 'bar';\n}\n``` ", + ( + "First code block:\n\n```python\ndef foo():\n return 'bar'\n```\n\n" + # This one has trailing whitespace after the second code block: + # https://github.com/simonw/llm/pull/718#issuecomment-2613177036 + "Second code block:\n\n```javascript\nfunction foo() {\n return 'bar';\n}\n``` " + ), True, "function foo() {\n return 'bar';\n}\n", ], @@ -268,8 +276,6 @@ def test_schema_dsl_multi(): ("Hello \n\t world!", 12, True, True, "Hello world!"), # Edge cases ("12345", 5, False, False, "12345"), - ("123456", 5, False, False, "12..."), - ("12345", 5, False, True, "12345"), # Unchanged for exact fit ("123456", 5, False, False, "12..."), # Regular truncation for small max_length # Very long string ("A" * 200, 10, False, False, "AAAAAAA..."), @@ -287,7 +293,6 @@ def test_schema_dsl_multi(): True, "12345...", ), # Too small for keep_end, use regular - ("1234567890", 9, False, True, "12... 90"), # Just enough for keep_end ], ) def test_truncate_string(text, max_length, normalize_whitespace, keep_end, expected): @@ -328,7 +333,7 @@ def test_test_truncate_string_keep_end( assert result == expected_full # Only check prefix/suffix when we expect truncation with keep_end - if prefix_len is not None and len(text) > max_length and max_length >= 9: + if prefix_len is not None and len(text) > max_length >= 9: assert result[:prefix_len] == text[:prefix_len] assert result[-prefix_len:] == text[-prefix_len:] assert "... " in result From da3de8d5985d2a96e6e4d8a0d0aaeafae7af6d2d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 16:20:31 -0700 Subject: [PATCH 121/258] Modernize code examples in docs Relates to Ruff work in #1556 --- docs/embeddings/python-api.md | 16 +++++++++------- docs/plugins/advanced-model-plugins.md | 19 +++++++++++-------- docs/plugins/plugin-hooks.md | 10 ++++++---- docs/plugins/tutorial-model-plugin.md | 14 +++++--------- docs/python-api.md | 16 +++++++++------- 5 files changed, 40 insertions(+), 35 deletions(-) diff --git a/docs/embeddings/python-api.md b/docs/embeddings/python-api.md index c6ec44e42..fe97da3db 100644 --- a/docs/embeddings/python-api.md +++ b/docs/embeddings/python-api.md @@ -13,8 +13,10 @@ vector = embedding_model.embed("my happy hound") ``` If the embedding model can handle binary input, you can call `.embed()` with a byte string instead. You can check the `supports_binary` property to see if this is supported: ```python +from pathlib import Path + if embedding_model.supports_binary: - vector = embedding_model.embed(open("my-image.jpg", "rb").read()) + vector = embedding_model.embed(Path("my-image.jpg").read_bytes()) ``` The `embedding_model.supports_text` property indicates if the model supports text input. @@ -122,12 +124,12 @@ A collection instance has the following properties and methods: - `model_id` - the string ID of the embedding model used for this collection - `model()` - returns the `EmbeddingModel` instance, based on that `model_id` - `count()` - returns the integer number of items in the collection -- `embed(id: str, text: str, metadata: dict=None, store: bool=False)` - embeds the given string and stores it in the collection under the given ID. Can optionally include metadata (stored as JSON) and store the text content itself in the database table. -- `embed_multi(entries: Iterable, store: bool=False, batch_size: int=100)` - see above -- `embed_multi_with_metadata(entries: Iterable, store: bool=False, batch_size: int=100)` - see above -- `similar(query: str, number: int=10)` - returns a list of entries that are most similar to the embedding of the given query string -- `similar_by_id(id: str, number: int=10)` - returns a list of entries that are most similar to the embedding of the item with the given ID -- `similar_by_vector(vector: List[float], number: int=10, skip_id: str=None)` - returns a list of entries that are most similar to the given embedding vector, optionally skipping the entry with the given ID +- `embed(id: str, value: str | bytes, metadata: dict[str, Any] | None = None, store: bool = False)` - embeds the given value and stores it in the collection under the given ID. Can optionally include metadata (stored as JSON) and store the text or binary content itself in the database table. +- `embed_multi(entries: Iterable[tuple[str, str | bytes]], store: bool = False, batch_size: int = 100)` - see above +- `embed_multi_with_metadata(entries: Iterable[tuple[str, str | bytes, dict[str, Any] | None]], store: bool = False, batch_size: int = 100)` - see above +- `similar(value: str | bytes, number: int = 10, prefix: str | None = None)` - returns a list of entries that are most similar to the embedding of the given value +- `similar_by_id(id: str, number: int = 10, prefix: str | None = None)` - returns a list of entries that are most similar to the embedding of the item with the given ID +- `similar_by_vector(vector: list[float], number: int = 10, skip_id: str | None = None, prefix: str | None = None)` - returns a list of entries that are most similar to the given embedding vector, optionally skipping the entry with the given ID - `delete()` - deletes the collection and its embeddings from the database There is also a `Collection.exists(db, name)` class method which returns a boolean value and can be used to determine if a collection exists or not in a database: diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index 3a1e2b939..45de8854a 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -58,7 +58,8 @@ The async version of a model subclasses `llm.AsyncModel` instead of `llm.Model`. This example shows a subset of the OpenAI default plugin illustrating how this method might work: ```python -from typing import AsyncGenerator +from collections.abc import AsyncGenerator + import llm class MyAsyncModel(llm.AsyncModel): @@ -168,17 +169,19 @@ See {ref}`the Python attachments documentation ` for det ### Specifying attachment types -A `Model` subclass can list the types of attachments it accepts by defining a `attachment_types` class attribute: +A `Model` subclass can list the types of attachments it accepts by defining an `attachment_types` class attribute: ```python class NewModel(llm.Model): model_id = "new-model" - attachment_types = { - "image/png", - "image/jpeg", - "image/webp", - "image/gif", - } + attachment_types = frozenset( + { + "image/png", + "image/jpeg", + "image/webp", + "image/gif", + } + ) ``` These content types are detected when an attachment is passed to LLM using `llm -a filename`, or can be specified by the user using the `--attachment-type filename image/png` option. diff --git a/docs/plugins/plugin-hooks.md b/docs/plugins/plugin-hooks.md index edd32837d..4b8b2922a 100644 --- a/docs/plugins/plugin-hooks.md +++ b/docs/plugins/plugin-hooks.md @@ -228,7 +228,7 @@ The `llm.Template` class has the following constructor: .. autoclass:: llm.Template ``` -The loader function should raise a `ValueError` if the template cannot be found or loaded correctly, providing a clear error message. +The loader function should raise a `ValueError` if the template cannot be found or loaded correctly, providing a clear error message. If you need to translate an exception from another library, catch that specific exception and use `raise ValueError(...) from ex` to preserve its context. Note that `functions:` provided by templates using this plugin hook will not be made available, to avoid the risk of plugin hooks that load templates from remote sources introducing arbitrary code execution vulnerabilities. @@ -275,11 +275,13 @@ def my_fragment_loader(argument: str) -> llm.Fragment: ) # Or for the case where you want to return multiple fragments and attachments: -def my_fragment_loader(argument: str) -> list[llm.Fragment]: +def my_fragment_loader( + argument: str, +) -> list[llm.Fragment | llm.Attachment]: "Docs go here." return [ - llm.Fragment("Fragment 1 content", "my-fragments:{argument}"), - llm.Fragment("Fragment 2 content", "my-fragments:{argument}"), + llm.Fragment("Fragment 1 content", f"my-fragments:{argument}"), + llm.Fragment("Fragment 2 content", f"my-fragments:{argument}"), llm.Attachment(path="/path/to/image.png"), ] ``` diff --git a/docs/plugins/tutorial-model-plugin.md b/docs/plugins/tutorial-model-plugin.md index 5de5e45c1..2c67cbcb0 100644 --- a/docs/plugins/tutorial-model-plugin.md +++ b/docs/plugins/tutorial-model-plugin.md @@ -348,18 +348,14 @@ The `delay` token will let us simulate a streaming language model, where tokens Options are defined using an inner class on the model, called `Options`. It should extend the `llm.Options` class. -First, add this import to the top of your `llm_markov.py` file: -```python -from typing import Optional -``` -Then add this `Options` class to your model: +Add this `Options` class to your model: ```python class Markov(Model): model_id = "markov" class Options(llm.Options): - length: Optional[int] = None - delay: Optional[float] = None + length: int | None = None + delay: float | None = None ``` Let's add extra validation rules to our options. Length must be at least 2. Duration must be between 0 and 10. @@ -376,11 +372,11 @@ We can now add Pydantic field validators for our two new rules, plus inline docu ```python class Options(llm.Options): - length: Optional[int] = Field( + length: int | None = Field( description="Number of words to generate", default=None ) - delay: Optional[float] = Field( + delay: float | None = Field( description="Seconds to delay between each token", default=None ) diff --git a/docs/python-api.md b/docs/python-api.md index a0929dbca..06f3f4b53 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -173,20 +173,20 @@ for response in chain.responses(): Pass a function to the `before_call=` parameter of `model.chain()` to have that called before every tool call is executed. You can raise `llm.CancelToolCall()` to cancel that tool call. -The method signature is `def before_call(tool: Optional[llm.Tool], tool_call: llm.ToolCall)` - that first `tool` argument can be `None` if the model requests a tool be executed that has not been provided in the `tools=` list. +The method signature is `def before_call(tool: llm.Tool | None, tool_call: llm.ToolCall)` - that first `tool` argument can be `None` if the model requests a tool be executed that has not been provided in the `tools=` list. Here's an example: ```python import llm -from typing import Optional def upper(text: str) -> str: "Convert text to uppercase." return text.upper() -def before_call(tool: Optional[llm.Tool], tool_call: llm.ToolCall): - print(f"About to call tool {tool.name} with arguments {tool_call.arguments}") - if tool.name == "upper" and "bad" in repr(tool_call.arguments): +def before_call(tool: llm.Tool | None, tool_call: llm.ToolCall): + tool_name = tool.name if tool is not None else tool_call.name + print(f"About to call tool {tool_name} with arguments {tool_call.arguments}") + if tool_name == "upper" and "bad" in repr(tool_call.arguments): raise llm.CancelToolCall("Not allowed to call upper on text containing 'bad'") model = llm.get_model("gpt-4.1-mini") @@ -460,11 +460,13 @@ print(model.prompt( The {ref}`fragment system ` from the CLI tool can also be accessed from the Python API, by passing `fragments=` and/or `system_fragments=` lists of strings to the `prompt()` method: ```python +from pathlib import Path + response = model.prompt( "What do these documents say about dogs?", fragments=[ - open("dogs1.txt").read(), - open("dogs2.txt").read(), + Path("dogs1.txt").read_text(), + Path("dogs2.txt").read_text(), ], system_fragments=[ "You answer questions like Snoopy", From 3bb8968c24aae8f9de57781b098756ce68ea975c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 16:52:30 -0700 Subject: [PATCH 122/258] Prototype content-addressed message store Adds llm/logs.py with a LogStore that keeps conversations as a parent-linked tree of messages. Each message is identified by a hash over its own canonical content plus its parent's hash, so conversations sharing a prefix share the rows that store it. Forking points a new thread at an existing message and copies nothing. A client that holds conversation state itself and re-sends the whole history each turn writes only the tail. Storage for a long conversation drops from quadratic to linear. The new tables sit alongside the existing ones, which are untouched, so previously logged records need no backfill. Attachments and fragments reuse the tables that already content-address them - identity is always the resolved content, but the bytes may live behind a reference. Round-trip is lossless for every part type, which the old schema could not manage: part ordering, provider metadata, redacted reasoning, server-executed tool calls, tool result exceptions and attachments. Co-Authored-By: Claude Opus 5 --- docs/logging.md | 72 +++++- llm/logs.py | 453 ++++++++++++++++++++++++++++++++ llm/migrations.py | 127 +++++++++ tests/test_logs_store.py | 543 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 1194 insertions(+), 1 deletion(-) create mode 100644 llm/logs.py create mode 100644 tests/test_logs_store.py diff --git a/docs/logging.md b/docs/logging.md index 4fba39f75..16e495ca8 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -300,7 +300,9 @@ cog.out("```sql\n") for table in ( "conversations", "schemas", "responses", "responses_fts", "attachments", "prompt_attachments", "fragments", "fragment_aliases", "prompt_fragments", "system_fragments", "tools", - "tool_responses", "tool_calls", "tool_results", "tool_instances" + "tool_responses", "tool_calls", "tool_results", "tool_instances", + "tool_results_attachments", + "messages", "parts", "part_attachments", "turns", "turn_tools", "threads", ): schema = db[table].schema cog.out(format(cleanup_sql(schema))) @@ -420,6 +422,74 @@ CREATE TABLE "tool_instances" ( "name" TEXT, "arguments" TEXT ); +CREATE TABLE "tool_results_attachments" ( + "tool_result_id" INTEGER REFERENCES "tool_results"("id"), + "attachment_id" TEXT REFERENCES "attachments"("id"), + "order" INTEGER, + PRIMARY KEY ("tool_result_id", + "attachment_id") +); +CREATE TABLE "messages" ( + "hash" TEXT PRIMARY KEY, + "parent_hash" TEXT REFERENCES "messages"("hash"), + "role" TEXT, + "provider_metadata" TEXT +); +CREATE TABLE "parts" ( + "id" INTEGER PRIMARY KEY, + "message_hash" TEXT REFERENCES "messages"("hash"), + "position" INTEGER, + "type" TEXT, + "text" TEXT, + "fragment_id" INTEGER REFERENCES "fragments"("id"), + "redacted" INTEGER, + "name" TEXT, + "arguments" TEXT, + "output" TEXT, + "tool_call_id" TEXT, + "server_executed" INTEGER, + "exception" TEXT, + "tool_id" INTEGER REFERENCES "tools"("id"), + "instance_id" INTEGER REFERENCES "tool_instances"("id"), + "provider_metadata" TEXT +); +CREATE TABLE "part_attachments" ( + "part_id" INTEGER REFERENCES "parts"("id"), + "attachment_id" TEXT REFERENCES "attachments"("id"), + "order" INTEGER, + PRIMARY KEY ("part_id", + "attachment_id") +); +CREATE TABLE "turns" ( + "id" TEXT PRIMARY KEY, + "thread_id" TEXT, + "parent_message_hash" TEXT REFERENCES "messages"("hash"), + "tip_message_hash" TEXT REFERENCES "messages"("hash"), + "model" TEXT, + "resolved_model" TEXT, + "options_json" TEXT, + "schema_id" TEXT REFERENCES "schemas"("id"), + "input_tokens" INTEGER, + "output_tokens" INTEGER, + "token_details" TEXT, + "duration_ms" INTEGER, + "datetime_utc" TEXT, + "response_json" TEXT, + "error" TEXT +); +CREATE TABLE "turn_tools" ( + "turn_id" TEXT REFERENCES "turns"("id"), + "tool_id" INTEGER REFERENCES "tools"("id"), + PRIMARY KEY ("turn_id", + "tool_id") +); +CREATE TABLE "threads" ( + "id" TEXT PRIMARY KEY, + "name" TEXT, + "tip_message_hash" TEXT REFERENCES "messages"("hash"), + "forked_from" TEXT REFERENCES "threads"("id"), + "datetime_utc" TEXT +); ``` `responses_fts` configures [SQLite full-text search](https://www.sqlite.org/fts5.html) against the `prompt` and `response` columns in the `responses` table. diff --git a/llm/logs.py b/llm/logs.py new file mode 100644 index 000000000..978f156a0 --- /dev/null +++ b/llm/logs.py @@ -0,0 +1,453 @@ +"""Content-addressed storage for conversation message trees. + +A conversation is a parent-linked chain of :class:`llm.Message` objects. +Each message is identified by a hash over its own canonical content plus +its parent's hash, so two conversations that share a prefix share the +rows that store it. Forking a conversation, or re-sending a history from +a client that holds the conversation state itself, both write only the +messages that are genuinely new. + +The identity of a message is its *resolved* content. Storage may still +be by reference — a text part sourced from a fragment stores a +``fragment_id`` rather than a second copy of the text, and attachments +reuse the existing content-addressed ``attachments`` table — but the +hash always covers the content as the model saw it. +""" + +import datetime +import hashlib +import json +from typing import Any + +from .migrations import migrate +from .models import Attachment +from .parts import ( + AttachmentPart, + Message, + ReasoningPart, + TextPart, + ToolCallPart, + ToolResultPart, +) +from .utils import ensure_tool, make_schema_id, monotonic_ulid + +__all__ = [ + "HASH_PREFIX", + "LogStore", + "canonical_json", + "content_hash", + "message_hash", +] + +# Hashes are tagged with the algorithm that produced them so a future +# change to the canonical form or the digest is detectable rather than +# silently splitting the dedup space into two incompatible halves. +HASH_PREFIX = "b2:" + +_DIGEST_SIZE = 16 + + +def canonical_json(obj: Any) -> str: + """Serialize to the canonical JSON form used for hashing. + + Keys sorted, no insignificant whitespace, non-ASCII left as-is. This + form is part of the documented contract: changing it changes every + hash. + """ + return json.dumps( + obj, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + + +def content_hash(obj: Any) -> str: + "Tagged hash of the canonical JSON form of ``obj``." + canonical = canonical_json(obj) + digest = hashlib.blake2b( + canonical.encode("utf-8"), digest_size=_DIGEST_SIZE + ).hexdigest() + return f"{HASH_PREFIX}{digest}" + + +def message_hash(message: Message, parent_hash: str | None) -> str: + """Identity of ``message`` when reached via ``parent_hash``. + + The parent participates, so the same content at a different point in + a conversation is a different node. That is what makes a shared + prefix collapse to shared rows without any explicit comparison. + """ + return content_hash({"parent": parent_hash, "message": message.to_dict()}) + + +class LogStore: + """Read and write conversation history in a SQLite database. + + Wraps a ``sqlite_utils.Database`` and applies any outstanding + migrations on construction, so a fresh database and an existing one + are handled the same way:: + + store = LogStore(sqlite_utils.Database("logs.db")) + """ + + def __init__(self, db): + self.db = db + migrate(db) + + # -- writing ------------------------------------------------------- + + def ensure_chain(self, messages, parent: str | None = None) -> str | None: + """Store ``messages`` as a chain and return the hash of the tip. + + Messages already present are left alone, so a caller that + re-sends a whole conversation - a client holding the state + itself, or a fork of an existing thread - writes only the + messages that are new. Passing ``parent`` appends to an existing + chain instead of starting a new one. + """ + tip = parent + for message in messages: + tip = self._ensure_message(message, tip) + return tip + + def _ensure_message(self, message: Message, parent_hash: str | None) -> str: + hash = message_hash(message, parent_hash) + if self.db["messages"].count_where("hash = ?", [hash]): + # Already stored - and because the hash covers the parent, + # everything below it is stored too. + return hash + with self.db.conn: + self.db["messages"].insert( + { + "hash": hash, + "parent_hash": parent_hash, + "role": message.role, + "provider_metadata": _dump(message.provider_metadata), + } + ) + for position, part in enumerate(message.parts): + self._write_part(hash, position, part) + return hash + + def _write_part(self, message_hash_: str, position: int, part) -> None: + row: dict[str, Any] = { + "message_hash": message_hash_, + "position": position, + "provider_metadata": _dump(getattr(part, "provider_metadata", None)), + } + attachments: list[Any] = [] + + if isinstance(part, TextPart): + row["type"] = "text" + row.update(self._text_columns(part.text)) + elif isinstance(part, ReasoningPart): + row["type"] = "reasoning" + row.update(self._text_columns(part.text)) + row["redacted"] = int(part.redacted) + elif isinstance(part, ToolCallPart): + row["type"] = "tool_call" + row["name"] = part.name + row["arguments"] = json.dumps(part.arguments) + row["tool_call_id"] = part.tool_call_id + row["server_executed"] = int(part.server_executed) + elif isinstance(part, ToolResultPart): + row["type"] = "tool_result" + row["name"] = part.name + row["output"] = part.output + row["tool_call_id"] = part.tool_call_id + row["server_executed"] = int(part.server_executed) + row["exception"] = part.exception + attachments = list(part.attachments) + elif isinstance(part, AttachmentPart): + row["type"] = "attachment" + if part.attachment is not None: + attachments = [part.attachment] + else: + raise TypeError(f"Cannot store {part!r}") + + part_id = self.db["parts"].insert(row).last_pk + for order, attachment in enumerate(attachments): + self.db["part_attachments"].insert( + { + "part_id": part_id, + "attachment_id": ensure_attachment(self.db, attachment), + "order": order, + } + ) + + def _text_columns(self, text: str) -> dict[str, Any]: + """Store text by reference when the same content is already a + fragment, otherwise inline. + + The hash always covers the resolved text either way - this only + decides where the bytes live. + """ + if text: + rows = list( + self.db.query( + "select id from fragments where hash = ?", + [hashlib.sha256(text.encode("utf-8")).hexdigest()], + ) + ) + if rows: + return {"text": None, "fragment_id": rows[0]["id"]} + return {"text": text, "fragment_id": None} + + # -- reading ------------------------------------------------------- + + def load_chain(self, tip: str | None) -> list[Message]: + """Return the full chain ending at ``tip``, oldest message first. + + Raises ``KeyError`` if ``tip`` is not in the store. + """ + if tip is None: + return [] + rows = [] + hash: str | None = tip + while hash is not None: + found = list(self.db.query("select * from messages where hash = ?", [hash])) + if not found: + raise KeyError(hash) + rows.append(found[0]) + hash = found[0]["parent_hash"] + rows.reverse() + parts_by_message = self._load_parts([row["hash"] for row in rows]) + return [ + Message( + role=row["role"], + parts=parts_by_message.get(row["hash"], []), + provider_metadata=_load(row["provider_metadata"]), + ) + for row in rows + ] + + def _load_parts(self, message_hashes: list[str]) -> dict[str, list[Any]]: + if not message_hashes: + return {} + placeholders = ",".join("?" * len(message_hashes)) + part_rows = list( + self.db.query( + f""" + select parts.*, fragments.content as fragment_content + from parts + left join fragments on parts.fragment_id = fragments.id + where parts.message_hash in ({placeholders}) + order by parts.message_hash, parts.position + """, + message_hashes, + ) + ) + attachments = self._load_part_attachments([row["id"] for row in part_rows]) + out: dict[str, list[Any]] = {} + for row in part_rows: + out.setdefault(row["message_hash"], []).append( + _part_from_row(row, attachments.get(row["id"], [])) + ) + return out + + def _load_part_attachments(self, part_ids: list[int]) -> dict[int, list[Any]]: + if not part_ids: + return {} + placeholders = ",".join("?" * len(part_ids)) + out: dict[int, list[Any]] = {} + for row in self.db.query( + f""" + select part_attachments.part_id, attachments.* + from part_attachments + join attachments on part_attachments.attachment_id = attachments.id + where part_attachments.part_id in ({placeholders}) + order by part_attachments.part_id, part_attachments."order" + """, + part_ids, + ): + out.setdefault(row["part_id"], []).append(Attachment.from_row(row)) + return out + + # -- threads ------------------------------------------------------- + + def create_thread( + self, + name: str | None = None, + tip: str | None = None, + forked_from: str | None = None, + ) -> str: + "Create a named pointer at a message and return its id." + thread_id = str(monotonic_ulid()).lower() + self.db["threads"].insert( + { + "id": thread_id, + "name": name, + "tip_message_hash": tip, + "forked_from": forked_from, + "datetime_utc": _now(), + } + ) + return thread_id + + def fork( + self, + message_hash_: str, + name: str | None = None, + forked_from: str | None = None, + ) -> str: + """Start a new thread from an existing message. + + Nothing is copied - the new thread points at a message that is + already stored, so its whole history is shared with the thread it + came from until the two diverge. + """ + if not self.db["messages"].count_where("hash = ?", [message_hash_]): + raise KeyError(message_hash_) + return self.create_thread(name=name, tip=message_hash_, forked_from=forked_from) + + def thread_tip(self, thread_id: str) -> str | None: + "The message a thread currently points at." + rows = list(self.db.query("select * from threads where id = ?", [thread_id])) + if not rows: + raise KeyError(thread_id) + return rows[0]["tip_message_hash"] + + def thread_messages(self, thread_id: str) -> list[Message]: + "The full history of a thread, oldest message first." + return self.load_chain(self.thread_tip(thread_id)) + + def append(self, thread_id: str, messages) -> str | None: + "Add messages to the end of a thread and return the new tip." + tip = self.ensure_chain(messages, parent=self.thread_tip(thread_id)) + self.db["threads"].update(thread_id, {"tip_message_hash": tip}) + return tip + + # -- turns --------------------------------------------------------- + + def log(self, response, thread_id: str | None = None) -> str: + """Record a completed response. + + The input chain and the response's own output are stored as + messages; everything that is specific to this particular call - + timings, usage, which model answered - goes on the turn, because + message rows are shared and so cannot carry provenance. + """ + parent = self.ensure_chain(response.prompt.messages) + tip = self.ensure_chain(response.messages(), parent=parent) + + schema_id = None + if response.prompt.schema: + schema_id, schema_json = make_schema_id(response.prompt.schema) + self.db["schemas"].insert( + {"id": schema_id, "content": schema_json}, ignore=True + ) + + turn_id = response.id or str(monotonic_ulid()).lower() + self.db["turns"].insert( + { + "id": turn_id, + "thread_id": thread_id, + "parent_message_hash": parent, + "tip_message_hash": tip, + "model": response.model.model_id, + "resolved_model": response.resolved_model, + "options_json": _dump( + { + key: value + for key, value in dict(response.prompt.options).items() + if value is not None + } + ), + "schema_id": schema_id, + "input_tokens": response.input_tokens, + "output_tokens": response.output_tokens, + "token_details": _dump(response.token_details), + "duration_ms": response.duration_ms(), + "datetime_utc": response.datetime_utc(), + "response_json": _dump(response.response_json), + }, + replace=True, + ) + for tool in response.prompt.tools: + self.db["turn_tools"].insert( + {"turn_id": turn_id, "tool_id": ensure_tool(self.db, tool)}, + replace=True, + ) + if thread_id is not None: + self.db["threads"].update(thread_id, {"tip_message_hash": tip}) + return turn_id + + # -- pending work -------------------------------------------------- + + def pending_tool_calls(self, tip: str | None) -> list[Any]: + """Tool calls at the tip of a chain that have no result yet. + + A chain ending in tool calls with nothing after them is a paused + conversation waiting to be resumed - it needs no separate record. + """ + chain = self.load_chain(tip) + if not chain: + return [] + return [part for part in chain[-1].parts if isinstance(part, ToolCallPart)] + + +def ensure_attachment(db, attachment) -> str: + "Store an attachment, returning its content-addressed id." + attachment_id = attachment.id() + db["attachments"].insert( + { + "id": attachment_id, + "type": attachment.resolve_type(), + "path": attachment.path, + "url": attachment.url, + "content": attachment.content, + }, + replace=True, + ) + return attachment_id + + +def _part_from_row(row: dict, attachments: list[Any]): + type = row["type"] + provider_metadata = _load(row["provider_metadata"]) + text = row["fragment_content"] if row["fragment_id"] else row["text"] + if type == "text": + return TextPart(text=text or "", provider_metadata=provider_metadata) + if type == "reasoning": + return ReasoningPart( + text=text or "", + redacted=bool(row["redacted"]), + provider_metadata=provider_metadata, + ) + if type == "tool_call": + return ToolCallPart( + name=row["name"] or "", + arguments=json.loads(row["arguments"] or "{}"), + tool_call_id=row["tool_call_id"], + server_executed=bool(row["server_executed"]), + provider_metadata=provider_metadata, + ) + if type == "tool_result": + return ToolResultPart( + name=row["name"] or "", + output=row["output"] or "", + tool_call_id=row["tool_call_id"], + server_executed=bool(row["server_executed"]), + exception=row["exception"], + attachments=attachments, + provider_metadata=provider_metadata, + ) + if type == "attachment": + return AttachmentPart( + attachment=attachments[0] if attachments else None, + provider_metadata=provider_metadata, + ) + raise ValueError(f"Unknown part type: {type!r}") + + +def _dump(value: dict | None) -> str | None: + return json.dumps(value) if value else None + + +def _load(value: str | None) -> dict | None: + return json.loads(value) if value else None + + +def _now() -> str: + return str(datetime.datetime.now(datetime.timezone.utc)) diff --git a/llm/migrations.py b/llm/migrations.py index b93e83014..50c44bb0d 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -426,3 +426,130 @@ def m022_response_reasoning(db): # NULL/empty when no reasoning was emitted or when the provider # only reported an opaque token count (the redacted-marker case). db["responses"].add_column("reasoning", str) + + +@migration +def m023_content_addressed_messages(db): + # The content-addressed message tree. A message's hash covers its own + # content *and* its parent's hash, so conversations sharing a prefix + # share the rows storing it. Nothing here replaces the older tables - + # they stay exactly as they are so existing logs need no backfill. + db["messages"].create( + { + "hash": str, + "parent_hash": str, + "role": str, + "provider_metadata": str, + }, + pk="hash", + foreign_keys=(("parent_hash", "messages", "hash"),), + ) + db["messages"].create_index(["parent_hash"]) + + # Parts are plain child rows rather than content-addressed in their + # own right: prefix sharing already dedupes at the message level, and + # the genuinely large payloads (attachments, fragments) live in + # tables that are content-addressed already. + db["parts"].create( + { + "id": int, + "message_hash": str, + "position": int, + "type": str, + # text / reasoning + "text": str, + "fragment_id": int, + "redacted": int, + # tool_call / tool_result + "name": str, + "arguments": str, + "output": str, + "tool_call_id": str, + "server_executed": int, + "exception": str, + "tool_id": int, + "instance_id": int, + "provider_metadata": str, + }, + pk="id", + foreign_keys=( + ("message_hash", "messages", "hash"), + ("fragment_id", "fragments", "id"), + ("tool_id", "tools", "id"), + ("instance_id", "tool_instances", "id"), + ), + ) + db["parts"].create_index(["message_hash", "position"], unique=True) + + # Covers both AttachmentPart and the attachments a tool result can + # carry, so there is one mechanism instead of two. + db["part_attachments"].create( + { + "part_id": int, + "attachment_id": str, + "order": int, + }, + pk=("part_id", "attachment_id"), + foreign_keys=( + ("part_id", "parts", "id"), + ("attachment_id", "attachments", "id"), + ), + ) + + # A turn is one call to a model. Provenance lives here rather than on + # the message rows, which are shared and so cannot carry it. + db["turns"].create( + { + "id": str, + "thread_id": str, + "parent_message_hash": str, + "tip_message_hash": str, + "model": str, + "resolved_model": str, + "options_json": str, + "schema_id": str, + "input_tokens": int, + "output_tokens": int, + "token_details": str, + "duration_ms": int, + "datetime_utc": str, + "response_json": str, + "error": str, + }, + pk="id", + foreign_keys=( + ("parent_message_hash", "messages", "hash"), + ("tip_message_hash", "messages", "hash"), + ("schema_id", "schemas", "id"), + ), + ) + + db["turn_tools"].create( + { + "turn_id": str, + "tool_id": int, + }, + pk=("turn_id", "tool_id"), + foreign_keys=( + ("turn_id", "turns", "id"), + ("tool_id", "tools", "id"), + ), + ) + + # A thread is a named, mutable pointer at a message - the only + # mutable thing in the new schema. Forking is a second pointer at an + # interior message. + db["threads"].create( + { + "id": str, + "name": str, + "tip_message_hash": str, + "forked_from": str, + "datetime_utc": str, + }, + pk="id", + foreign_keys=( + ("tip_message_hash", "messages", "hash"), + ("forked_from", "threads", "id"), + ), + ) diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py new file mode 100644 index 000000000..410512378 --- /dev/null +++ b/tests/test_logs_store.py @@ -0,0 +1,543 @@ +"""Tests for llm.logs — the content-addressed message store. + +The store keeps conversations as a parent-linked tree of messages, each +identified by a hash over its own content plus its parent's hash. Shared +prefixes are stored once, so forking a conversation and re-sending a +history from a stateless client both write only what is new. +""" + +import pytest +import sqlite_utils + +import llm +from llm.logs import LogStore, canonical_json, message_hash +from llm.models import Attachment +from llm.parts import ( + AttachmentPart, + Message, + ReasoningPart, + TextPart, + ToolCallPart, + ToolResultPart, +) +from llm.utils import ensure_fragment + +NEW_TABLES = { + "messages", + "parts", + "part_attachments", + "turns", + "turn_tools", + "threads", +} + +# Tables the pre-existing logging path writes to. The new store must +# leave every one of them alone, so old logs stay readable without a +# backfill. +LEGACY_TABLES = { + "conversations", + "responses", + "attachments", + "prompt_attachments", + "fragments", + "tools", + "tool_calls", + "tool_results", +} + + +@pytest.fixture +def store(): + return LogStore(sqlite_utils.Database(memory=True)) + + +# ---- canonical form + hashing ---------------------------------------- + + +class TestCanonicalJson: + def test_key_order_does_not_matter(self): + assert canonical_json({"b": 1, "a": 2}) == canonical_json({"a": 2, "b": 1}) + + def test_is_compact(self): + assert canonical_json({"a": 1, "b": 2}) == '{"a":1,"b":2}' + + def test_non_ascii_is_not_escaped(self): + assert canonical_json({"a": "é"}) == '{"a":"é"}' + + def test_nested_keys_are_sorted(self): + assert canonical_json({"a": {"z": 1, "y": 2}}) == '{"a":{"y":2,"z":1}}' + + +class TestMessageHash: + def test_is_deterministic(self): + message = llm.user("Hello") + assert message_hash(message, None) == message_hash(message, None) + + def test_carries_algorithm_prefix(self): + assert message_hash(llm.user("Hello"), None).startswith("b2:") + + def test_equal_content_hashes_equal(self): + assert message_hash(llm.user("Hello"), None) == message_hash( + llm.user("Hello"), None + ) + + def test_different_text_hashes_differently(self): + assert message_hash(llm.user("Hello"), None) != message_hash( + llm.user("Goodbye"), None + ) + + def test_role_participates(self): + assert message_hash(llm.user("Hello"), None) != message_hash( + llm.assistant("Hello"), None + ) + + def test_parent_participates(self): + message = llm.user("Hello") + root = message_hash(message, None) + assert message_hash(message, root) != root + + def test_part_order_participates(self): + one = Message(role="user", parts=[TextPart(text="a"), TextPart(text="b")]) + two = Message(role="user", parts=[TextPart(text="b"), TextPart(text="a")]) + assert message_hash(one, None) != message_hash(two, None) + + def test_provider_metadata_participates(self): + plain = Message(role="assistant", parts=[TextPart(text="Hi")]) + with_meta = Message( + role="assistant", + parts=[TextPart(text="Hi", provider_metadata={"openai": {"id": "rs_1"}})], + ) + assert message_hash(plain, None) != message_hash(with_meta, None) + + def test_provider_metadata_key_order_does_not_matter(self): + one = Message( + role="assistant", + parts=[TextPart(text="Hi", provider_metadata={"a": 1, "b": 2})], + ) + two = Message( + role="assistant", + parts=[TextPart(text="Hi", provider_metadata={"b": 2, "a": 1})], + ) + assert message_hash(one, None) == message_hash(two, None) + + def test_redacted_reasoning_differs_from_empty_reasoning(self): + redacted = Message(role="assistant", parts=[ReasoningPart(redacted=True)]) + empty = Message(role="assistant", parts=[ReasoningPart()]) + assert message_hash(redacted, None) != message_hash(empty, None) + + +# ---- schema ---------------------------------------------------------- + + +class TestSchema: + def test_creates_new_tables(self, store): + assert NEW_TABLES <= set(store.db.table_names()) + + def test_leaves_legacy_tables_in_place(self, store): + assert LEGACY_TABLES <= set(store.db.table_names()) + + def test_migrating_twice_is_a_noop(self, store): + before = store.db.schema + LogStore(store.db) + assert store.db.schema == before + + def test_messages_are_keyed_by_hash(self, store): + assert store.db["messages"].pks == ["hash"] + + def test_parts_are_ordered_within_a_message(self, store): + indexes = {tuple(index.columns) for index in store.db["parts"].indexes} + assert ("message_hash", "position") in indexes + + +# ---- chain round-trip ------------------------------------------------ + + +def round_trip(store, messages): + "Write a chain, read it straight back." + return store.load_chain(store.ensure_chain(messages)) + + +class TestChainRoundTrip: + def test_empty_chain_has_no_tip(self, store): + assert store.ensure_chain([]) is None + + def test_load_chain_of_none_is_empty(self, store): + assert store.load_chain(None) == [] + + def test_tip_is_the_hash_of_the_last_message(self, store): + messages = [llm.user("Hi"), llm.assistant("Hello")] + tip = store.ensure_chain(messages) + root = message_hash(messages[0], None) + assert tip == message_hash(messages[1], root) + + def test_single_message(self, store): + messages = [llm.user("Hi")] + assert round_trip(store, messages) == messages + + def test_multiple_turns(self, store): + messages = [ + llm.system("Be brief"), + llm.user("Hi"), + llm.assistant("Hello"), + llm.user("Again"), + llm.assistant("Hello again"), + ] + assert round_trip(store, messages) == messages + + def test_reasoning_including_redacted(self, store): + messages = [ + llm.user("Think"), + llm.assistant( + ReasoningPart(redacted=True), + ReasoningPart(text="Considering it"), + TextPart(text="Done"), + ), + ] + assert round_trip(store, messages) == messages + + def test_interleaved_parts_keep_their_order(self, store): + # The ordering the old schema could not express: reasoning, a + # tool call, more reasoning, then text, all in one message. + messages = [ + llm.user("Search"), + llm.assistant( + ReasoningPart(text="first"), + ToolCallPart(name="search", arguments={"q": "a"}, tool_call_id="tc1"), + ReasoningPart(text="second"), + TextPart(text="answer"), + ToolCallPart(name="search", arguments={"q": "b"}, tool_call_id="tc2"), + ), + ] + assert round_trip(store, messages) == messages + + def test_tool_call_with_server_executed(self, store): + messages = [ + llm.assistant( + ToolCallPart( + name="web_search", + arguments={"query": "pelicans"}, + tool_call_id="tc1", + server_executed=True, + ) + ) + ] + assert round_trip(store, messages) == messages + + def test_tool_result_with_exception(self, store): + messages = [ + llm.tool_message( + ToolResultPart( + name="lookup", + output="", + tool_call_id="tc1", + exception="ValueError: nope", + ) + ) + ] + assert round_trip(store, messages) == messages + + def test_provider_metadata_survives(self, store): + messages = [ + Message( + role="assistant", + parts=[ + ReasoningPart( + redacted=True, + provider_metadata={ + "openai": {"id": "rs_1", "encrypted_content": "xyz"} + }, + ) + ], + provider_metadata={"openai": {"response_id": "resp_1"}}, + ) + ] + assert round_trip(store, messages) == messages + + def test_attachment_part(self, store): + messages = [ + llm.user( + "Describe", + Attachment(type="image/png", content=b"fake-png-bytes"), + ) + ] + assert round_trip(store, messages) == messages + + def test_attachment_part_with_provider_metadata(self, store): + messages = [ + llm.user( + AttachmentPart( + attachment=Attachment(type="image/png", content=b"bytes"), + provider_metadata={"openai": {"file_id": "file_1"}}, + ) + ) + ] + assert round_trip(store, messages) == messages + + def test_attachment_part_without_an_attachment(self, store): + messages = [llm.user(AttachmentPart())] + assert round_trip(store, messages) == messages + + def test_tool_result_attachments_keep_their_order(self, store): + messages = [ + llm.tool_message( + ToolResultPart( + name="render", + output="two images", + tool_call_id="tc1", + attachments=[ + Attachment(type="image/png", content=b"first"), + Attachment(type="image/png", content=b"second"), + ], + ) + ) + ] + assert round_trip(store, messages) == messages + + def test_attachment_content_is_shared_with_legacy_table(self, store): + store.ensure_chain( + [llm.user("Describe", Attachment(type="image/png", content=b"bytes"))] + ) + assert store.db["attachments"].count == 1 + + def test_unknown_tip_raises(self, store): + with pytest.raises(KeyError): + store.load_chain("b2:does-not-exist") + + +# ---- storage by reference -------------------------------------------- + + +class TestStorageByReference: + def test_text_matching_a_fragment_is_stored_by_reference(self, store): + content = "a large reusable fragment" + ensure_fragment(store.db, content) + store.ensure_chain([llm.user(content)]) + row = next(iter(store.db["parts"].rows)) + assert row["text"] is None + assert row["fragment_id"] is not None + + def test_fragment_backed_text_still_round_trips(self, store): + content = "a large reusable fragment" + ensure_fragment(store.db, content) + messages = [llm.user(content)] + assert round_trip(store, messages) == messages + + def test_fragment_backed_text_hashes_the_same_as_inline(self, store): + content = "a large reusable fragment" + inline_tip = store.ensure_chain([llm.user(content)]) + ensure_fragment(store.db, content) + by_reference_tip = store.ensure_chain([llm.user(content)]) + # Identity is the resolved text, so where the bytes live makes + # no difference to the hash - and the second write is a no-op. + assert inline_tip == by_reference_tip + assert store.db["messages"].count == 1 + + +# ---- dedup ----------------------------------------------------------- + + +class TestDedup: + def test_writing_the_same_chain_twice_adds_nothing(self, store): + messages = [llm.user("Hi"), llm.assistant("Hello")] + first = store.ensure_chain(messages) + second = store.ensure_chain(messages) + assert first == second + assert store.db["messages"].count == 2 + assert store.db["parts"].count == 2 + + def test_extending_a_chain_only_writes_the_new_message(self, store): + messages = [llm.user("Hi"), llm.assistant("Hello")] + store.ensure_chain(messages) + store.ensure_chain(messages + [llm.user("More")]) + assert store.db["messages"].count == 3 + + def test_stateless_client_resending_history_writes_only_the_tail(self, store): + # What an OpenAI Chat Completions style server sees: the client + # holds the conversation and posts the whole thing every turn. + history = [] + for turn in range(5): + history.append(llm.user(f"question {turn}")) + history.append(llm.assistant(f"answer {turn}")) + store.ensure_chain(history) + assert store.db["messages"].count == 10 + + def test_diverging_chains_share_their_prefix(self, store): + prefix = [llm.user("Hi"), llm.assistant("Hello")] + store.ensure_chain(prefix + [llm.user("left")]) + store.ensure_chain(prefix + [llm.user("right")]) + # Two shared messages plus one for each branch. + assert store.db["messages"].count == 4 + + def test_appending_to_a_known_tip_skips_the_prefix_entirely(self, store): + tip = store.ensure_chain([llm.user("Hi"), llm.assistant("Hello")]) + new_tip = store.ensure_chain([llm.user("More")], parent=tip) + assert store.db["messages"].count == 3 + assert [message.parts[0].text for message in store.load_chain(new_tip)] == [ + "Hi", + "Hello", + "More", + ] + + def test_same_content_under_a_different_parent_is_a_different_message(self, store): + store.ensure_chain([llm.user("Hi"), llm.assistant("Same")]) + store.ensure_chain([llm.user("Different"), llm.assistant("Same")]) + assert store.db["messages"].count == 4 + + +# ---- threads and forking --------------------------------------------- + + +class TestThreads: + def test_new_thread_has_no_tip(self, store): + thread_id = store.create_thread(name="Empty") + assert store.thread_messages(thread_id) == [] + + def test_appending_advances_the_tip(self, store): + thread_id = store.create_thread(name="Chat") + store.append(thread_id, [llm.user("Hi")]) + store.append(thread_id, [llm.assistant("Hello")]) + assert [message.role for message in store.thread_messages(thread_id)] == [ + "user", + "assistant", + ] + + def test_thread_records_its_name(self, store): + thread_id = store.create_thread(name="Named") + assert store.db["threads"].get(thread_id)["name"] == "Named" + + def test_unknown_thread_raises(self, store): + with pytest.raises(KeyError): + store.append("nope", [llm.user("Hi")]) + + +class TestForking: + def test_fork_shares_history_up_to_the_fork_point(self, store): + thread_id = store.create_thread(name="Original") + store.append(thread_id, [llm.user("Hi"), llm.assistant("Hello")]) + fork_point = store.append(thread_id, [llm.user("Original branch")]) + + forked = store.fork(fork_point, name="What if") + assert [message.parts[0].text for message in store.thread_messages(forked)] == [ + "Hi", + "Hello", + "Original branch", + ] + + def test_forking_writes_no_new_messages(self, store): + thread_id = store.create_thread() + tip = store.append(thread_id, [llm.user("Hi"), llm.assistant("Hello")]) + before = store.db["messages"].count + store.fork(tip, name="Copy") + assert store.db["messages"].count == before + + def test_forked_branches_diverge_without_copying_the_prefix(self, store): + original = store.create_thread(name="Original") + fork_point = store.append(original, [llm.user("Hi"), llm.assistant("Hello")]) + forked = store.fork(fork_point, name="Alternative") + + store.append(original, [llm.user("down one path")]) + store.append(forked, [llm.user("down another")]) + + # Two shared messages, plus one new message per branch. + assert store.db["messages"].count == 4 + assert len(store.thread_messages(original)) == 3 + assert len(store.thread_messages(forked)) == 3 + + def test_fork_records_where_it_came_from(self, store): + original = store.create_thread(name="Original") + tip = store.append(original, [llm.user("Hi")]) + forked = store.fork(tip, name="Copy", forked_from=original) + assert store.db["threads"].get(forked)["forked_from"] == original + + def test_fork_of_an_interior_message_drops_the_later_history(self, store): + thread_id = store.create_thread() + fork_point = store.append(thread_id, [llm.user("Hi")]) + store.append(thread_id, [llm.assistant("Hello"), llm.user("More")]) + forked = store.fork(fork_point) + assert len(store.thread_messages(forked)) == 1 + + +# ---- pending tool calls ---------------------------------------------- + + +class TestPendingToolCalls: + def test_trailing_tool_calls_are_pending(self, store): + tip = store.ensure_chain( + [ + llm.user("Search"), + llm.assistant( + ToolCallPart(name="search", arguments={}, tool_call_id="tc1") + ), + ] + ) + assert [call.tool_call_id for call in store.pending_tool_calls(tip)] == ["tc1"] + + def test_tool_calls_followed_by_results_are_not_pending(self, store): + tip = store.ensure_chain( + [ + llm.user("Search"), + llm.assistant( + ToolCallPart(name="search", arguments={}, tool_call_id="tc1") + ), + llm.tool_message( + ToolResultPart(name="search", output="done", tool_call_id="tc1") + ), + ] + ) + assert store.pending_tool_calls(tip) == [] + + def test_a_plain_reply_has_nothing_pending(self, store): + tip = store.ensure_chain([llm.user("Hi"), llm.assistant("Hello")]) + assert store.pending_tool_calls(tip) == [] + + +# ---- logging a response ---------------------------------------------- + + +class TestLogResponse: + def test_writes_a_turn(self, store, mock_model): + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + turn_id = store.log(response) + assert store.db["turns"].get(turn_id)["model"] == "mock" + + def test_turn_records_usage(self, store, mock_model): + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi there") + response.text() + turn = store.db["turns"].get(store.log(response)) + assert turn["input_tokens"] == 2 + assert turn["output_tokens"] == 1 + + def test_turn_spans_from_its_parent_to_its_tip(self, store, mock_model): + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + turn = store.db["turns"].get(store.log(response)) + chain = store.load_chain(turn["tip_message_hash"]) + assert [message.role for message in chain] == ["user", "assistant"] + assert chain[-1].parts[0].text == "Hello" + + def test_logging_advances_the_thread(self, store, mock_model): + thread_id = store.create_thread(name="Chat") + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + store.log(response, thread_id=thread_id) + assert [ + message.parts[0].text for message in store.thread_messages(thread_id) + ] == [ + "Hi", + "Hello", + ] + + def test_logging_the_same_response_twice_is_idempotent(self, store, mock_model): + # A turn is identified by the response it records, so re-logging + # one updates it in place rather than duplicating the event. + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + assert store.log(response) == store.log(response) + assert store.db["messages"].count == 2 + assert store.db["turns"].count == 1 From 5a4644a95480e0319a5c854f32d1effb8a7cd3e8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 16:59:54 -0700 Subject: [PATCH 123/258] CI against sqlite-utils <4 and >=4, refs #1555 (#1558) --- .github/workflows/test.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e3ab350af..901330dd2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,10 @@ jobs: include: - os: ubuntu-latest python-version: "3.14" - sqlite-utils-version: "4.0rc4" + sqlite-utils-version: "<4" + - os: ubuntu-latest + python-version: "3.14" + sqlite-utils-version: ">=4" steps: - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} @@ -28,15 +31,15 @@ jobs: - name: Install dependencies run: | pip install . --group dev - - name: Install sqlite-utils pre-release + - name: Install sqlite-utils ${{ matrix.sqlite-utils-version }} if: matrix.sqlite-utils-version != '' run: | - pip install sqlite-utils==${{ matrix.sqlite-utils-version }} + pip install 'sqlite-utils${{ matrix.sqlite-utils-version }}' - name: Run tests run: | python -m pytest -vv - name: Check if cog needs to be run - if: matrix.sqlite-utils-version == '4.0rc4' + if: matrix.sqlite-utils-version == '>=4' run: | cog --check \ -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" \ From cd8cd3f90c8e4713572ecf3972ab941deb65f27d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 17:12:02 -0700 Subject: [PATCH 124/258] Wire the CLI into the content-addressed store Prompts now dual-write: the legacy tables exactly as before, plus the new message tree via LogStore. Every existing read path keeps working while the new schema is unproven. Threads reuse the conversation id, so the two identifier spaces line up. That needs the CLI to settle on a conversation before prompting rather than letting the legacy logger invent one at write time and discard it, so a plain one-off prompt now runs through a conversation too. `llm -c` takes its history from the new tables when a thread exists. That chain is the exact message list that was sent and returned, so reasoning signatures and provider metadata survive a reload - the rebuild from legacy columns can only approximate it. Conversations logged before this schema existed have no thread and fall back to the old path. Co-Authored-By: Claude Opus 5 --- llm/cli.py | 33 +++++++- llm/logs.py | 30 ++++++- llm/models.py | 26 +++++- tests/test_logs_store.py | 178 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 259 insertions(+), 8 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index ecd8121a7..e359792c4 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -62,6 +62,7 @@ ) from llm.models import ChainResponse, _BaseConversation +from .logs import LogStore from .migrations import migrate from .plugins import load_plugins, pm from .utils import ( @@ -813,7 +814,11 @@ def read_prompt(): click.echo(render_model_with_options(model_id, async_=async_)) return - if conversation is None and (tools or python_tools): + if conversation is None: + # Always work through a conversation, even for a one-off prompt. + # The legacy logger invents one anyway and throws the id away; + # creating it here means both writers agree on which conversation + # (and so which thread) this response belongs to. conversation = model.conversation() if conversation: @@ -986,6 +991,7 @@ async def inner(): response = asyncio.run(response.to_sync_response()) # At this point ALL forms should have a log_to_db() method that works: response.log_to_db(db) + log_to_store(db, response) @cli.command() @@ -1303,9 +1309,23 @@ def chat( show_reasoning=not hide_reasoning, ) response.log_to_db(db) + log_to_store(db, response) print() +def log_to_store(db, response): + """Mirror a response into the content-addressed tables. + + Written alongside the legacy tables, not instead of them, so every + existing read path keeps working while the new schema beds in. + """ + store = LogStore(db) + for item in getattr(response, "_responses", None) or [response]: + if isinstance(item, AsyncResponse): + item = asyncio.run(item.to_sync_response()) + store.log(item) + + def load_conversation( conversation_id: str | None, async_=False, @@ -1345,6 +1365,17 @@ def load_conversation( + list(response_obj.prompt.messages) ) conversation.responses.append(response_obj) + + # If this conversation has a thread in the content-addressed tables, + # take the history from there. That chain is the exact message list + # that was sent and returned, so reasoning signatures and provider + # metadata survive - unlike the rebuild above, which can only work + # from the flattened legacy columns. + try: + conversation.loaded_messages = LogStore(db).thread_messages(conversation_id) + except KeyError: + pass + return conversation diff --git a/llm/logs.py b/llm/logs.py index 978f156a0..d958e1f72 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -20,7 +20,7 @@ from typing import Any from .migrations import migrate -from .models import Attachment +from .models import Attachment, _conversation_name from .parts import ( AttachmentPart, Message, @@ -271,9 +271,10 @@ def create_thread( name: str | None = None, tip: str | None = None, forked_from: str | None = None, + id: str | None = None, ) -> str: "Create a named pointer at a message and return its id." - thread_id = str(monotonic_ulid()).lower() + thread_id = id or str(monotonic_ulid()).lower() self.db["threads"].insert( { "id": thread_id, @@ -285,6 +286,17 @@ def create_thread( ) return thread_id + def ensure_thread(self, thread_id: str, name: str | None = None) -> str: + """Return the thread with this id, creating it if it is new. + + Threads created from a conversation reuse the conversation's id, + so the two identifier spaces line up while both sets of tables + are being written. + """ + if not self.db["threads"].count_where("id = ?", [thread_id]): + self.create_thread(name=name, id=thread_id) + return thread_id + def fork( self, message_hash_: str, @@ -328,8 +340,20 @@ def log(self, response, thread_id: str | None = None) -> str: timings, usage, which model answered - goes on the turn, because message rows are shared and so cannot carry provenance. """ + if thread_id is None: + conversation = getattr(response, "conversation", None) + if conversation is not None: + thread_id = self.ensure_thread( + conversation.id, + name=_conversation_name( + response.prompt.prompt or response.prompt.system or "" + ), + ) + parent = self.ensure_chain(response.prompt.messages) - tip = self.ensure_chain(response.messages(), parent=parent) + # _messages_now() rather than messages(), which is a coroutine on + # AsyncResponse. + tip = self.ensure_chain(response._messages_now(), parent=parent) schema_id = None if response.prompt.schema: diff --git a/llm/models.py b/llm/models.py index ff58d6366..249e19696 100644 --- a/llm/models.py +++ b/llm/models.py @@ -563,12 +563,25 @@ class _BaseConversation: responses: list["_BaseResponse"] = field(default_factory=list) tools: list[ToolDef] | None = None chain_limit: int | None = None + # History read back from storage, used as the chain for the next turn + # when this conversation has not yet produced a response in this + # process. Unlike a chain rebuilt from logged responses this is the + # exact message list, so reasoning signatures and provider metadata + # survive being reloaded. + loaded_messages: list[Any] | None = None @classmethod @abstractmethod def from_row(cls, row: Any) -> "_BaseConversation": raise NotImplementedError + def _record_response(self, response: "_BaseResponse") -> None: + "Record a completed response as part of this conversation." + self.responses.append(response) + # History now comes from the live responses, so anything read + # back from storage is superseded. + self.loaded_messages = None + def _build_full_chain( self, prompt: str | None, @@ -602,7 +615,12 @@ def _build_full_chain( return list(explicit_messages) chain: list[Any] = [] - if self.responses: + if self.loaded_messages: + # Storage holds the exact chain, so prefer it over anything + # rebuilt from logged responses. Cleared as soon as this + # conversation produces a response of its own. + chain.extend(self.loaded_messages) + elif self.responses: last = self.responses[-1] # last.prompt.messages already contains the full input chain # under the invariant, so use the last response only and then @@ -2013,7 +2031,7 @@ def __iter__(self) -> Iterator[str]: yield text if self.conversation: - self.conversation.responses.append(self) + self.conversation._record_response(self) self._end = time.monotonic() self._done = True self._on_done() @@ -2039,7 +2057,7 @@ def stream_events(self): yield self._stream_events[-1] if self.conversation: - self.conversation.responses.append(self) + self.conversation._record_response(self) self._end = time.monotonic() self._done = True self._on_done() @@ -2440,7 +2458,7 @@ def _ensure_async_generator(self): async def _async_finalize(self): if self.conversation: - self.conversation.responses.append(self) + self.conversation._record_response(self) self._end = time.monotonic() self._done = True if hasattr(self, "_generator"): diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 410512378..6c2dd4b0a 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -8,8 +8,10 @@ import pytest import sqlite_utils +from click.testing import CliRunner import llm +from llm.cli import cli from llm.logs import LogStore, canonical_json, message_hash from llm.models import Attachment from llm.parts import ( @@ -541,3 +543,179 @@ def test_logging_the_same_response_twice_is_idempotent(self, store, mock_model): assert store.log(response) == store.log(response) assert store.db["messages"].count == 2 assert store.db["turns"].count == 1 + + +# ---- conversations map onto threads ---------------------------------- + + +class TestConversationThreads: + def test_log_uses_the_conversation_id_as_the_thread_id(self, store, mock_model): + conversation = mock_model.conversation() + mock_model.enqueue(["Hello"]) + response = conversation.prompt("Hi") + response.text() + store.log(response) + assert store.db["threads"].get(conversation.id) is not None + + def test_successive_turns_extend_the_same_thread(self, store, mock_model): + conversation = mock_model.conversation() + for reply in ("Hello", "Hello again"): + mock_model.enqueue([reply]) + response = conversation.prompt("Hi") + response.text() + store.log(response) + assert store.db["threads"].count == 1 + assert [ + message.parts[0].text for message in store.thread_messages(conversation.id) + ] == ["Hi", "Hello", "Hi", "Hello again"] + + def test_a_response_without_a_conversation_creates_no_thread( + self, store, mock_model + ): + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + store.log(response) + assert store.db["threads"].count == 0 + + def test_an_explicit_thread_id_wins(self, store, mock_model): + thread_id = store.create_thread(name="Mine") + conversation = mock_model.conversation() + mock_model.enqueue(["Hello"]) + response = conversation.prompt("Hi") + response.text() + store.log(response, thread_id=thread_id) + assert store.db["threads"].count == 1 + assert len(store.thread_messages(thread_id)) == 2 + + +# ---- CLI integration ------------------------------------------------- + + +@pytest.fixture +def cli_store(user_path): + "A LogStore over the same database the CLI logs to." + return LogStore(sqlite_utils.Database(str(user_path / "logs.db"))) + + +def run(*args): + result = CliRunner().invoke(cli, list(args), catch_exceptions=False) + assert result.exit_code == 0, result.output + return result + + +class TestCliDualWrite: + def test_a_prompt_writes_a_turn(self, cli_store): + run("-m", "echo", "Hi") + assert cli_store.db["turns"].count == 1 + + def test_a_prompt_still_writes_the_legacy_tables(self, cli_store): + run("-m", "echo", "Hi") + assert cli_store.db["responses"].count == 1 + + def test_the_turn_points_at_the_stored_chain(self, cli_store): + run("-m", "echo", "Hi") + turn = next(iter(cli_store.db["turns"].rows)) + chain = cli_store.load_chain(turn["tip_message_hash"]) + assert [message.role for message in chain] == ["user", "assistant"] + + def test_the_thread_matches_the_conversation(self, cli_store): + run("-m", "echo", "Hi") + conversation_id = next(iter(cli_store.db["conversations"].rows))["id"] + assert cli_store.thread_tip(conversation_id) is not None + + def test_no_log_writes_nothing(self, cli_store): + run("-m", "echo", "Hi", "--no-log") + assert cli_store.db["turns"].count == 0 + assert cli_store.db["messages"].count == 0 + + +class TestCliContinuation: + def test_continuing_extends_the_same_thread(self, cli_store): + run("-m", "echo", "First") + run("-m", "echo", "Second", "-c") + assert cli_store.db["threads"].count == 1 + conversation_id = next(iter(cli_store.db["conversations"].rows))["id"] + assert len(cli_store.thread_messages(conversation_id)) == 4 + + def test_history_comes_from_the_new_tables(self, user_path): + # Delete the legacy rows the old continuation path reads from. If + # `-c` still sends the full history, it can only have come from + # the content-addressed tables. + run("-m", "echo", "First") + + db = sqlite_utils.Database(str(user_path / "logs.db")) + conversation_id = next(iter(db["conversations"].rows))["id"] + with db.conn: + db.execute("delete from responses") + db.close() + + run("-m", "echo", "Second", "-c") + + # Four messages only if the second turn was built on top of the + # first. Had the history been lost, the second turn would have + # started a fresh root and the thread would hold just two. + store = LogStore(sqlite_utils.Database(str(user_path / "logs.db"))) + chain = store.thread_messages(conversation_id) + assert len(chain) == 4 + assert chain[0].parts[0].text == "First" + + def test_continuing_writes_only_the_new_messages(self, cli_store): + run("-m", "echo", "First") + before = cli_store.db["messages"].count + run("-m", "echo", "Second", "-c") + assert cli_store.db["messages"].count == before + 2 + + +# ---- history loaded from storage ------------------------------------- + + +class TestLoadedMessages: + def test_loaded_messages_supply_the_history(self, mock_model): + conversation = mock_model.conversation() + conversation.loaded_messages = [llm.user("Earlier"), llm.assistant("Reply")] + mock_model.enqueue(["Next"]) + response = conversation.prompt("Now") + response.text() + assert [message.parts[0].text for message in response.prompt.messages] == [ + "Earlier", + "Reply", + "Now", + ] + + def test_a_completed_response_supersedes_them(self, mock_model): + conversation = mock_model.conversation() + conversation.loaded_messages = [llm.user("Earlier"), llm.assistant("Reply")] + mock_model.enqueue(["First"]) + conversation.prompt("One").text() + mock_model.enqueue(["Second"]) + response = conversation.prompt("Two") + response.text() + # The live response takes over; the loaded history is not + # replayed a second time. + assert [message.parts[0].text for message in response.prompt.messages] == [ + "Earlier", + "Reply", + "One", + "First", + "Two", + ] + + +class TestLegacyConversations: + def test_continuing_a_conversation_with_no_thread_still_works(self, user_path): + # Conversations logged before this schema existed have no thread, + # so `-c` has to fall back to rebuilding from the legacy rows. + run("-m", "echo", "First") + + path = str(user_path / "logs.db") + db = sqlite_utils.Database(path) + with db.conn: + db.execute("delete from threads") + db.execute("delete from turns") + db.execute("delete from parts") + db.execute("delete from messages") + db.close() + + result = run("-m", "echo", "Second", "-c") + assert "First" in result.output From 62a09e3615d30cb0f6592cf189f62f05c8f1b020 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 17:42:19 -0700 Subject: [PATCH 125/258] Write the message store from log_to_db, not from the CLI log_to_db() is what plugins call, so mirroring into the content-addressed tables belongs there. Wiring it into llm/cli.py meant anything using the library API - llm-coding-agent calls chain.log_to_db() directly - wrote only the legacy tables. That was worse than leaving the new tables empty. Since load_conversation prefers a thread's chain over a rebuild from logged responses, a turn written by a legacy-only caller was absent from the thread, and the next `llm -c` built its prompt from the stale thread and silently dropped that turn. Both writers agreed before the store became authoritative on read. LogStore is now constructed at the top of log_to_db, because it applies migrations and those have to run before the legacy inserts create any tables implicitly. Co-Authored-By: Claude Opus 5 --- llm/cli.py | 15 -------------- llm/models.py | 12 ++++++++++++ tests/test_logs_store.py | 42 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index e359792c4..43ba9ae8b 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -991,7 +991,6 @@ async def inner(): response = asyncio.run(response.to_sync_response()) # At this point ALL forms should have a log_to_db() method that works: response.log_to_db(db) - log_to_store(db, response) @cli.command() @@ -1309,23 +1308,9 @@ def chat( show_reasoning=not hide_reasoning, ) response.log_to_db(db) - log_to_store(db, response) print() -def log_to_store(db, response): - """Mirror a response into the content-addressed tables. - - Written alongside the legacy tables, not instead of them, so every - existing read path keeps working while the new schema beds in. - """ - store = LogStore(db) - for item in getattr(response, "_responses", None) or [response]: - if isinstance(item, AsyncResponse): - item = asyncio.run(item.to_sync_response()) - store.log(item) - - def load_conversation( conversation_id: str | None, async_=False, diff --git a/llm/models.py b/llm/models.py index 249e19696..1a29f6f1e 100644 --- a/llm/models.py +++ b/llm/models.py @@ -1431,6 +1431,12 @@ def token_usage(self) -> str: ) def log_to_db(self, db): + # Built up front because it applies migrations, which have to run + # before the inserts below create any tables implicitly. + from .logs import LogStore + + store = LogStore(db) + conversation = self.conversation if not conversation: conversation = Conversation(model=self.model) @@ -1629,6 +1635,12 @@ def log_to_db(self, db): }, ) + # Mirror into the content-addressed tables. This lives here + # rather than in the CLI because log_to_db() is what plugins + # call - anything that logs a response should populate both + # representations, not just `llm` itself. + store.log(self) + def _response_to_dict(response: "_BaseResponse") -> ResponseDict: """Shared serializer for Response.to_dict / AsyncResponse.to_dict. diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 6c2dd4b0a..f9bb9336d 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -719,3 +719,45 @@ def test_continuing_a_conversation_with_no_thread_still_works(self, user_path): result = run("-m", "echo", "Second", "-c") assert "First" in result.output + + +# ---- logging through the library API --------------------------------- + + +class TestLibraryLogging: + """`log_to_db` is what plugins call, so the store write belongs there + rather than in the CLI - otherwise anything that is not `llm` itself + writes only the legacy tables.""" + + def test_log_to_db_writes_the_store_too(self, store, mock_model): + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + response.log_to_db(store.db) + assert store.db["turns"].count == 1 + assert store.db["messages"].count == 2 + + def test_log_to_db_still_writes_the_legacy_tables(self, store, mock_model): + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + response.log_to_db(store.db) + assert store.db["responses"].count == 1 + + def test_a_chain_writes_the_store_too(self, store, mock_model): + conversation = mock_model.conversation() + mock_model.enqueue(["Hello"]) + chain = conversation.chain("Hi") + chain.text() + chain.log_to_db(store.db) + assert store.db["turns"].count == 1 + assert len(store.thread_messages(conversation.id)) == 2 + + def test_successive_library_turns_extend_the_thread(self, store, mock_model): + conversation = mock_model.conversation() + for reply in ("One", "Two"): + mock_model.enqueue([reply]) + response = conversation.prompt("Ask") + response.text() + response.log_to_db(store.db) + assert len(store.thread_messages(conversation.id)) == 4 From 29c465383cf094ed96892c4a978ebeece1945149 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 22:34:15 -0700 Subject: [PATCH 126/258] Reshape the message store around part payloads parts drops from 16 columns to 6. A part now stores Part.to_dict() as a payload, so reading is Part.from_dict() and a new part type or field needs no schema change - the lossless round-trip becomes true by construction rather than held up by a hand-written bijection between the wire form and a column per field. type and tool_name remain as write-time projections for querying; the read path ignores them. Storage is by reference, which is the point of the fragments feature: ask a hundred questions about a novel and the novel is stored once. Text that borrows from a fragment stores an ordered list of fragment ids and literals; attachments store an id into the existing attachments table. Identity is unaffected - hashing happens over the resolved content before anything is written. The previous fragment_id column never worked. It matched text against fragment hashes exactly, so it fired only when a prompt was empty, and it could not express a part built from several fragments. part_fragments and turn_fragments make "everything that used fragment X" a single indexed join, restoring what llm logs -f needs. turns drops response_json: a verbatim copy of the output is redundant with the structured parts and it was the only fat column in a table meant to be scanned. LogStore.verify() re-hashes every stored message. Reads now resolve references, so a reconstruction bug would produce a chain that differs from the one that was hashed and nothing else would notice - the wrong text would simply be sent to the model. Also fixes Conversation.prompt building its chain from the raw prompt text, omitting fragments, while Model.prompt included them. Since the CLI always goes through a conversation, every logged chain was missing its fragment content and llm -c would have resent history without it. m023 shipped only in alphas and mirrors data the legacy tables still hold in full, so m024 drops and recreates rather than carrying a data migration for a schema nobody has. Co-Authored-By: Claude Opus 5 --- conversation-views.sql | 152 ++++++++++++++ docs/logging.md | 52 ++--- llm/logs.py | 419 +++++++++++++++++++++++++++------------ llm/migrations.py | 164 +++++++++++++++ llm/models.py | 16 +- tests/test_logs_store.py | 212 +++++++++++++++++--- 6 files changed, 832 insertions(+), 183 deletions(-) create mode 100644 conversation-views.sql diff --git a/conversation-views.sql b/conversation-views.sql new file mode 100644 index 000000000..bec9d595d --- /dev/null +++ b/conversation-views.sql @@ -0,0 +1,152 @@ +-- A readable rendering of the content-addressed message tree. +-- +-- Prototype: applied directly to logs.db rather than added to +-- llm/migrations.py, so it can be reshaped without a migration. +-- Re-runnable - the view is dropped first. +-- +-- select entry from conversation_tree; -- everything +-- select entry from conversation_tree where id = 'faafcc3b'; +-- +-- One row per part, depth-first, so reading `entry` top to bottom +-- replays the conversation. Indentation tracks *branching*, not depth: +-- a conversation that never forks stays flush left however long it +-- runs, and each divergence steps one level in. Where a message forked, +-- each branch is headed [n/total] and its whole subtree is aligned +-- underneath, so it is always clear which reply belongs to which try. + +DROP VIEW IF EXISTS conversation_tree; +DROP VIEW IF EXISTS part_text; + +-- Resolves a part's text back from storage. Text that borrowed from a +-- fragment is stored as an ordered list of fragment references and +-- literals rather than a copy, so reading it means splicing the +-- fragment contents back in. +CREATE VIEW part_text AS +SELECT + p.id AS part_id, + coalesce( + json_extract(p.payload, '$.text'), + ( + SELECT group_concat( + coalesce( + json_extract(piece.value, '$.literal'), + (SELECT f.content FROM fragments f + WHERE f.id = json_extract(piece.value, '$.fragment')) + ), + '' ORDER BY piece.key + ) + FROM json_each(json_extract(p.payload, '$.text_ref')) piece + ) + ) AS text +FROM parts p; + +CREATE VIEW conversation_tree AS +WITH RECURSIVE +-- Where each message sits among its siblings. Computed up front +-- because window functions are not allowed in a recursive term. +sibling AS ( + SELECT + hash, + parent_hash, + row_number() OVER (PARTITION BY parent_hash ORDER BY rowid) AS ord, + count(*) OVER (PARTITION BY parent_hash) AS of + FROM messages +), +walk(root_hash, message_hash, depth, indent, ord, of, sort_key) AS ( + SELECT m.hash, m.hash, 0, 0, 1, 1, printf('%08d', m.rowid) + FROM messages m + WHERE m.parent_hash IS NULL + UNION ALL + SELECT + w.root_hash, + s.hash, + w.depth + 1, + -- Step in only where the parent actually forked. + w.indent + (CASE WHEN s.of > 1 THEN 1 ELSE 0 END), + s.ord, + s.of, + w.sort_key || '/' || printf('%08d', m.rowid) + FROM sibling s + JOIN messages m ON m.hash = s.hash + JOIN walk w ON s.parent_hash = w.message_hash +), +rendered AS ( + SELECT + w.*, + m.role, + p.position, + p.type, + -- Prefix: role for plain text, role + kind for anything else, so + -- a reasoning block or a tool call is never mistaken for what + -- the model actually said. + m.role + || CASE + WHEN p.type IS NULL OR p.type = 'text' THEN '' + ELSE ' ' || p.type + END + || ': ' AS prefix, + CASE p.type + WHEN 'text' THEN coalesce(pt.text, '') + WHEN 'reasoning' THEN + CASE + WHEN json_extract(p.payload, '$.redacted') + THEN '(reasoning withheld by provider)' + ELSE coalesce(pt.text, '') + END + WHEN 'tool_call' THEN + p.tool_name + || '(' || coalesce(json_extract(p.payload, '$.arguments'), '') || ')' + WHEN 'tool_result' THEN + p.tool_name + || ' -> ' || coalesce(json_extract(p.payload, '$.output'), '') + WHEN 'attachment' THEN '(attachment)' + ELSE coalesce(p.type, '(no content)') + END AS body + FROM walk w + JOIN messages m ON m.hash = w.message_hash + LEFT JOIN parts p ON p.message_hash = m.hash + LEFT JOIN part_text pt ON pt.part_id = p.id +), +margined AS ( + SELECT + r.*, + CASE + WHEN r.indent = 0 THEN '' + ELSE replace(hex(zeroblob((r.indent - 1) * 8)), '00', ' ') + -- The branch marker occupies the last indent step, so + -- the head of a branch and its descendants line up. + || CASE + WHEN r.of > 1 AND coalesce(r.position, 0) = 0 + THEN printf('%-8s', '[' || r.ord || '/' || r.of || ']') + ELSE ' ' + END + END AS margin + FROM rendered r +) +SELECT + -- Short, typeable handle for the whole tree. Every message reachable + -- from one root shares it, so filtering on it gives that + -- conversation and every branch of it. + substr(m.root_hash, 4, 8) AS id, + m.margin + || m.prefix + -- Wrapped lines sit under the prefix, so a multi-line answer + -- stays inside its own column. + || replace( + m.body, + char(10), + char(10) || replace( + hex(zeroblob(length(m.margin) + length(m.prefix))), '00', ' ' + ) + ) AS entry, + m.depth, + m.indent, + CASE WHEN m.of > 1 THEN m.ord ELSE NULL END AS branch, + m.of AS branches, + m.role, + m.type, + m.message_hash, + m.root_hash, + m.sort_key +FROM margined m +ORDER BY m.sort_key, m.position; diff --git a/docs/logging.md b/docs/logging.md index 16e495ca8..3fda94100 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -302,7 +302,8 @@ for table in ( "fragments", "fragment_aliases", "prompt_fragments", "system_fragments", "tools", "tool_responses", "tool_calls", "tool_results", "tool_instances", "tool_results_attachments", - "messages", "parts", "part_attachments", "turns", "turn_tools", "threads", + "messages", "parts", "part_attachments", "part_fragments", + "threads", "turns", "turn_tools", "turn_fragments", ): schema = db[table].schema cog.out(format(cleanup_sql(schema))) @@ -440,18 +441,8 @@ CREATE TABLE "parts" ( "message_hash" TEXT REFERENCES "messages"("hash"), "position" INTEGER, "type" TEXT, - "text" TEXT, - "fragment_id" INTEGER REFERENCES "fragments"("id"), - "redacted" INTEGER, - "name" TEXT, - "arguments" TEXT, - "output" TEXT, - "tool_call_id" TEXT, - "server_executed" INTEGER, - "exception" TEXT, - "tool_id" INTEGER REFERENCES "tools"("id"), - "instance_id" INTEGER REFERENCES "tool_instances"("id"), - "provider_metadata" TEXT + "tool_name" TEXT, + "payload" TEXT ); CREATE TABLE "part_attachments" ( "part_id" INTEGER REFERENCES "parts"("id"), @@ -460,9 +451,24 @@ CREATE TABLE "part_attachments" ( PRIMARY KEY ("part_id", "attachment_id") ); +CREATE TABLE "part_fragments" ( + "part_id" INTEGER REFERENCES "parts"("id"), + "fragment_id" INTEGER REFERENCES "fragments"("id"), + "order" INTEGER, + PRIMARY KEY ("part_id", + "fragment_id", + "order") +); +CREATE TABLE "threads" ( + "id" TEXT PRIMARY KEY, + "name" TEXT, + "tip_message_hash" TEXT REFERENCES "messages"("hash"), + "forked_from" TEXT REFERENCES "threads"("id"), + "datetime_utc" TEXT +); CREATE TABLE "turns" ( "id" TEXT PRIMARY KEY, - "thread_id" TEXT, + "thread_id" TEXT REFERENCES "threads"("id"), "parent_message_hash" TEXT REFERENCES "messages"("hash"), "tip_message_hash" TEXT REFERENCES "messages"("hash"), "model" TEXT, @@ -473,9 +479,7 @@ CREATE TABLE "turns" ( "output_tokens" INTEGER, "token_details" TEXT, "duration_ms" INTEGER, - "datetime_utc" TEXT, - "response_json" TEXT, - "error" TEXT + "datetime_utc" TEXT ); CREATE TABLE "turn_tools" ( "turn_id" TEXT REFERENCES "turns"("id"), @@ -483,12 +487,14 @@ CREATE TABLE "turn_tools" ( PRIMARY KEY ("turn_id", "tool_id") ); -CREATE TABLE "threads" ( - "id" TEXT PRIMARY KEY, - "name" TEXT, - "tip_message_hash" TEXT REFERENCES "messages"("hash"), - "forked_from" TEXT REFERENCES "threads"("id"), - "datetime_utc" TEXT +CREATE TABLE "turn_fragments" ( + "turn_id" TEXT REFERENCES "turns"("id"), + "fragment_id" INTEGER REFERENCES "fragments"("id"), + "order" INTEGER, + "kind" TEXT, + PRIMARY KEY ("turn_id", + "fragment_id", + "kind") ); ``` diff --git a/llm/logs.py b/llm/logs.py index d958e1f72..18331a38e 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -7,11 +7,13 @@ a client that holds the conversation state itself, both write only the messages that are genuinely new. -The identity of a message is its *resolved* content. Storage may still -be by reference — a text part sourced from a fragment stores a -``fragment_id`` rather than a second copy of the text, and attachments -reuse the existing content-addressed ``attachments`` table — but the -hash always covers the content as the model saw it. +The identity of a message is its *resolved* content, but storage is by +reference: text that borrows from a fragment stores the fragment's id in +place of a copy, and attachments store an id into the existing +content-addressed ``attachments`` table. Ask a hundred questions about a +novel and the novel is stored once. Reading resolves the references +again, so the hash always covers the content as the model saw it - +``LogStore.verify()`` re-derives it to prove that stays true. """ import datetime @@ -24,12 +26,11 @@ from .parts import ( AttachmentPart, Message, - ReasoningPart, - TextPart, + Part, ToolCallPart, ToolResultPart, ) -from .utils import ensure_tool, make_schema_id, monotonic_ulid +from .utils import ensure_fragment, ensure_tool, make_schema_id, monotonic_ulid __all__ = [ "HASH_PREFIX", @@ -97,7 +98,12 @@ def __init__(self, db): # -- writing ------------------------------------------------------- - def ensure_chain(self, messages, parent: str | None = None) -> str | None: + def ensure_chain( + self, + messages, + parent: str | None = None, + fragments=None, + ) -> str | None: """Store ``messages`` as a chain and return the hash of the tip. Messages already present are left alone, so a caller that @@ -105,13 +111,36 @@ def ensure_chain(self, messages, parent: str | None = None) -> str | None: itself, or a fork of an existing thread - writes only the messages that are new. Passing ``parent`` appends to an existing chain instead of starting a new one. + + ``fragments`` is an optional list of fragment contents that may + appear inside these messages. Any that do are stored as a + reference rather than a copy, which is the point of the fragments + feature: ask a hundred questions about a novel and the novel is + stored once. It never affects the hashes - identity is always the + resolved text. """ + fragment_map = self._fragment_map(fragments) tip = parent for message in messages: - tip = self._ensure_message(message, tip) + tip = self._ensure_message(message, tip, fragment_map) return tip - def _ensure_message(self, message: Message, parent_hash: str | None) -> str: + def _fragment_map(self, fragments) -> dict[str, int]: + "Map fragment content to its id, registering any that are new." + if not fragments: + return {} + return { + str(fragment): ensure_fragment(self.db, fragment) + for fragment in fragments + if str(fragment) + } + + def _ensure_message( + self, + message: Message, + parent_hash: str | None, + fragment_map: dict[str, int], + ) -> str: hash = message_hash(message, parent_hash) if self.db["messages"].count_where("hash = ?", [hash]): # Already stored - and because the hash covers the parent, @@ -127,72 +156,60 @@ def _ensure_message(self, message: Message, parent_hash: str | None) -> str: } ) for position, part in enumerate(message.parts): - self._write_part(hash, position, part) + self._write_part(hash, position, part, fragment_map) return hash - def _write_part(self, message_hash_: str, position: int, part) -> None: - row: dict[str, Any] = { - "message_hash": message_hash_, - "position": position, - "provider_metadata": _dump(getattr(part, "provider_metadata", None)), - } - attachments: list[Any] = [] - - if isinstance(part, TextPart): - row["type"] = "text" - row.update(self._text_columns(part.text)) - elif isinstance(part, ReasoningPart): - row["type"] = "reasoning" - row.update(self._text_columns(part.text)) - row["redacted"] = int(part.redacted) - elif isinstance(part, ToolCallPart): - row["type"] = "tool_call" - row["name"] = part.name - row["arguments"] = json.dumps(part.arguments) - row["tool_call_id"] = part.tool_call_id - row["server_executed"] = int(part.server_executed) - elif isinstance(part, ToolResultPart): - row["type"] = "tool_result" - row["name"] = part.name - row["output"] = part.output - row["tool_call_id"] = part.tool_call_id - row["server_executed"] = int(part.server_executed) - row["exception"] = part.exception - attachments = list(part.attachments) - elif isinstance(part, AttachmentPart): - row["type"] = "attachment" - if part.attachment is not None: - attachments = [part.attachment] + def _write_part( + self, + message_hash_: str, + position: int, + part, + fragment_map: dict[str, int], + ) -> None: + payload = part.to_dict() + attachments = _attachments_of(part) + + # Large content out of the payload and into the tables that + # already store it once: fragments for text, attachments for + # bytes. Both are resolved again on the way back out. + used_fragments = _encode_text_refs(payload, fragment_map) + if attachments: + attachment_ids = [ + ensure_attachment(self.db, attachment) for attachment in attachments + ] + _encode_attachment_refs(payload, attachment_ids) else: - raise TypeError(f"Cannot store {part!r}") + attachment_ids = [] - part_id = self.db["parts"].insert(row).last_pk - for order, attachment in enumerate(attachments): + part_id = ( + self.db["parts"] + .insert( + { + "message_hash": message_hash_, + "position": position, + "type": payload["type"], + "tool_name": payload.get("name"), + "payload": canonical_json(payload), + } + ) + .last_pk + ) + for order, attachment_id in enumerate(attachment_ids): self.db["part_attachments"].insert( { "part_id": part_id, - "attachment_id": ensure_attachment(self.db, attachment), + "attachment_id": attachment_id, "order": order, } ) - - def _text_columns(self, text: str) -> dict[str, Any]: - """Store text by reference when the same content is already a - fragment, otherwise inline. - - The hash always covers the resolved text either way - this only - decides where the bytes live. - """ - if text: - rows = list( - self.db.query( - "select id from fragments where hash = ?", - [hashlib.sha256(text.encode("utf-8")).hexdigest()], - ) + for order, fragment_id in enumerate(used_fragments): + self.db["part_fragments"].insert( + { + "part_id": part_id, + "fragment_id": fragment_id, + "order": order, + } ) - if rows: - return {"text": None, "fragment_id": rows[0]["id"]} - return {"text": text, "fragment_id": None} # -- reading ------------------------------------------------------- @@ -229,40 +246,55 @@ def _load_parts(self, message_hashes: list[str]) -> dict[str, list[Any]]: part_rows = list( self.db.query( f""" - select parts.*, fragments.content as fragment_content - from parts - left join fragments on parts.fragment_id = fragments.id - where parts.message_hash in ({placeholders}) - order by parts.message_hash, parts.position + select * from parts + where message_hash in ({placeholders}) + order by message_hash, position """, message_hashes, ) ) - attachments = self._load_part_attachments([row["id"] for row in part_rows]) + payloads = [json.loads(row["payload"]) for row in part_rows] + # Resolve the references put in on the way in. Both lookups are + # batched across the whole chain rather than done per part. + fragments = self._load_fragments(payloads) + attachments = self._load_attachments(payloads) out: dict[str, list[Any]] = {} - for row in part_rows: - out.setdefault(row["message_hash"], []).append( - _part_from_row(row, attachments.get(row["id"], [])) - ) + for row, payload in zip(part_rows, payloads): + _decode_text_refs(payload, fragments) + # Strip the attachment references before rebuilding, then + # hang the resolved objects back on. + ids = _attachment_ids(payload) + payload.pop("attachment", None) + payload.pop("attachments", None) + part = Part.from_dict(payload) + _resolve_attachments(part, ids, attachments) + out.setdefault(row["message_hash"], []).append(part) return out - def _load_part_attachments(self, part_ids: list[int]) -> dict[int, list[Any]]: - if not part_ids: + def _load_fragments(self, payloads: list[dict]) -> dict[int, str]: + ids = sorted({id for payload in payloads for id in _fragment_ids(payload)}) + if not ids: return {} - placeholders = ",".join("?" * len(part_ids)) - out: dict[int, list[Any]] = {} - for row in self.db.query( - f""" - select part_attachments.part_id, attachments.* - from part_attachments - join attachments on part_attachments.attachment_id = attachments.id - where part_attachments.part_id in ({placeholders}) - order by part_attachments.part_id, part_attachments."order" - """, - part_ids, - ): - out.setdefault(row["part_id"], []).append(Attachment.from_row(row)) - return out + placeholders = ",".join("?" * len(ids)) + return { + row["id"]: row["content"] + for row in self.db.query( + f"select id, content from fragments where id in ({placeholders})", + ids, + ) + } + + def _load_attachments(self, payloads: list[dict]) -> dict[str, Any]: + ids = sorted({id for payload in payloads for id in _attachment_ids(payload)}) + if not ids: + return {} + placeholders = ",".join("?" * len(ids)) + return { + row["id"]: Attachment.from_row(row) + for row in self.db.query( + f"select * from attachments where id in ({placeholders})", ids + ) + } # -- threads ------------------------------------------------------- @@ -350,7 +382,13 @@ def log(self, response, thread_id: str | None = None) -> str: ), ) - parent = self.ensure_chain(response.prompt.messages) + prompt_fragments = list(response.prompt.fragments or []) + system_fragments = list(response.prompt.system_fragments or []) + + parent = self.ensure_chain( + response.prompt.messages, + fragments=prompt_fragments + system_fragments, + ) # _messages_now() rather than messages(), which is a coroutine on # AsyncResponse. tip = self.ensure_chain(response._messages_now(), parent=parent) @@ -384,7 +422,6 @@ def log(self, response, thread_id: str | None = None) -> str: "token_details": _dump(response.token_details), "duration_ms": response.duration_ms(), "datetime_utc": response.datetime_utc(), - "response_json": _dump(response.response_json), }, replace=True, ) @@ -393,10 +430,61 @@ def log(self, response, thread_id: str | None = None) -> str: {"turn_id": turn_id, "tool_id": ensure_tool(self.db, tool)}, replace=True, ) + # Which fragments this call was given - provenance, so it belongs + # on the turn rather than on the shared message rows. This is what + # answers "show me everything that used fragment X". + for kind, fragments in ( + ("prompt", prompt_fragments), + ("system", system_fragments), + ): + for order, fragment in enumerate(fragments): + self.db["turn_fragments"].insert( + { + "turn_id": turn_id, + "fragment_id": ensure_fragment(self.db, fragment), + "order": order, + "kind": kind, + }, + replace=True, + ) if thread_id is not None: self.db["threads"].update(thread_id, {"tip_message_hash": tip}) return turn_id + # -- verification -------------------------------------------------- + + def verify(self) -> list[str]: + """Re-hash every stored message and return those that disagree. + + Reads resolve references - fragment text and attachment bytes are + stitched back in - so a bug there, or a fragment deleted out from + under a message, produces a chain that differs from the one that + was hashed. Nothing else would notice: the wrong text would just + be silently sent to the model. Re-deriving the hash from what + comes back out catches the whole class. + + An empty list means every message on disk still resolves to the + content its hash was taken over. + """ + broken = [] + for row in self.db.query("select hash, parent_hash from messages"): + parts = self._load_parts([row["hash"]]).get(row["hash"], []) + message_row = next( + iter( + self.db.query( + "select * from messages where hash = ?", [row["hash"]] + ) + ) + ) + message = Message( + role=message_row["role"], + parts=parts, + provider_metadata=_load(message_row["provider_metadata"]), + ) + if message_hash(message, row["parent_hash"]) != row["hash"]: + broken.append(row["hash"]) + return broken + # -- pending work -------------------------------------------------- def pending_tool_calls(self, tip: str | None) -> list[Any]: @@ -427,42 +515,115 @@ def ensure_attachment(db, attachment) -> str: return attachment_id -def _part_from_row(row: dict, attachments: list[Any]): - type = row["type"] - provider_metadata = _load(row["provider_metadata"]) - text = row["fragment_content"] if row["fragment_id"] else row["text"] - if type == "text": - return TextPart(text=text or "", provider_metadata=provider_metadata) - if type == "reasoning": - return ReasoningPart( - text=text or "", - redacted=bool(row["redacted"]), - provider_metadata=provider_metadata, - ) - if type == "tool_call": - return ToolCallPart( - name=row["name"] or "", - arguments=json.loads(row["arguments"] or "{}"), - tool_call_id=row["tool_call_id"], - server_executed=bool(row["server_executed"]), - provider_metadata=provider_metadata, - ) - if type == "tool_result": - return ToolResultPart( - name=row["name"] or "", - output=row["output"] or "", - tool_call_id=row["tool_call_id"], - server_executed=bool(row["server_executed"]), - exception=row["exception"], - attachments=attachments, - provider_metadata=provider_metadata, - ) - if type == "attachment": - return AttachmentPart( - attachment=attachments[0] if attachments else None, - provider_metadata=provider_metadata, +# -- reference encoding ------------------------------------------------ +# +# A stored payload is Part.to_dict() with large content swapped for a +# reference: fragment ids in place of text, attachment ids in place of +# bytes. Resolving it reproduces the wire form exactly, which is what +# makes it safe for the hash to be taken over the resolved content and +# never over what is on disk. + + +def _attachments_of(part) -> list[Any]: + "The Attachment objects a part carries, in order." + if isinstance(part, ToolResultPart): + return list(part.attachments) + if isinstance(part, AttachmentPart) and part.attachment is not None: + return [part.attachment] + return [] + + +def _encode_text_refs(payload: dict, fragment_map: dict[str, int]) -> list[int]: + """Replace ``text`` with a ``text_ref`` list of fragments and + literals. Returns the fragment ids used, in order. + + Nothing is replaced unless a fragment actually occurs in the text, so + a part that borrows no fragments keeps its plain ``text`` key. + """ + text = payload.get("text") + if not text or not fragment_map: + return [] + pieces: list[dict] = [] + used: list[int] = [] + remaining = text + while remaining: + # Earliest occurrence wins; on a tie the longer fragment does, so + # a fragment that is a prefix of another cannot mask it. + best: tuple[int, str] | None = None + for content in fragment_map: + index = remaining.find(content) + if index == -1: + continue + if best is None or (index, -len(content)) < (best[0], -len(best[1])): + best = (index, content) + if best is None: + pieces.append({"literal": remaining}) + break + index, content = best + if index: + pieces.append({"literal": remaining[:index]}) + pieces.append({"fragment": fragment_map[content]}) + used.append(fragment_map[content]) + remaining = remaining[index + len(content) :] + if not used: + return [] + del payload["text"] + payload["text_ref"] = pieces + return used + + +def _decode_text_refs(payload: dict, fragments: dict[int, str]) -> None: + "Reverse of _encode_text_refs, restoring the exact original text." + pieces = payload.pop("text_ref", None) + if pieces is None: + return + payload["text"] = "".join( + ( + piece["literal"] + if "literal" in piece + else fragments.get(piece["fragment"], "") ) - raise ValueError(f"Unknown part type: {type!r}") + for piece in pieces + ) + + +def _fragment_ids(payload: dict) -> list[int]: + return [ + piece["fragment"] + for piece in payload.get("text_ref") or [] + if "fragment" in piece + ] + + +def _encode_attachment_refs(payload: dict, attachment_ids: list[str]) -> None: + "Replace inline attachment dicts with their content-addressed ids." + if payload["type"] == "attachment": + payload["attachment"] = {"id": attachment_ids[0]} + elif payload["type"] == "tool_result": + payload["attachments"] = [{"id": id} for id in attachment_ids] + + +def _resolve_attachments(part, ids: list[str], attachments: dict[str, Any]) -> None: + """Hang the resolved Attachment objects back on a part. + + Done after ``Part.from_dict`` rather than by putting them back into + the payload, so the bytes are never round-tripped through base64 and + the objects keep the content-addressed id they were stored under. + """ + if not ids: + return + if isinstance(part, AttachmentPart): + part.attachment = attachments[ids[0]] + elif isinstance(part, ToolResultPart): + part.attachments = [attachments[id] for id in ids] + + +def _attachment_ids(payload: dict) -> list[str]: + if payload["type"] == "attachment" and "attachment" in payload: + return [payload["attachment"]["id"]] + if payload["type"] == "tool_result": + return [ref["id"] for ref in payload.get("attachments") or []] + return [] def _dump(value: dict | None) -> str | None: diff --git a/llm/migrations.py b/llm/migrations.py index 50c44bb0d..e2b1c3408 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -553,3 +553,167 @@ def m023_content_addressed_messages(db): ("forked_from", "threads", "id"), ), ) + + +@migration +def m024_message_store_payloads(db): + # Reshape the m023 tables. Parts now carry the wire form of the part + # as a payload rather than a column per field, so a new part type or + # field needs no schema change and reading is Part.from_dict(). + # + # The payload stores large content by reference - fragment ids for + # text, attachment ids for binary - which is the whole point of the + # fragments feature: a novel is stored once and pointed at from every + # prompt about it. Hashing is unaffected either way, because identity + # is computed over the resolved content before anything is written. + # + # m023 shipped only in alphas and its tables are a mirror of data the + # legacy tables still hold in full, so this drops and recreates + # rather than carrying a data migration for a schema nobody has. + for table in ( + "turn_tools", + "turn_fragments", + "turns", + "threads", + "part_attachments", + "part_fragments", + "parts", + "messages", + ): + db[table].drop(ignore=True) + + db["messages"].create( + { + "hash": str, + "parent_hash": str, + "role": str, + "provider_metadata": str, + }, + pk="hash", + foreign_keys=(("parent_hash", "messages", "hash"),), + ) + # Needed by the recursive descent through the tree and by the + # child count that identifies a fork. + db["messages"].create_index(["parent_hash"]) + + db["parts"].create( + { + "id": int, + "message_hash": str, + "position": int, + # text | reasoning | tool_call | tool_result | attachment + "type": str, + # Tool name for tool_call and tool_result parts, else NULL. + "tool_name": str, + # Part.to_dict(), with large content replaced by references. + # Authoritative: reading is Part.from_dict(resolved payload). + # type and tool_name above are write-time projections for + # querying; the read path ignores them. + "payload": str, + }, + pk="id", + foreign_keys=(("message_hash", "messages", "hash"),), + ) + # The hot read path, and it enforces one part per position. + db["parts"].create_index(["message_hash", "position"], unique=True) + + # Attachment and fragment ids are in the payload too. These tables + # exist so referential integrity, garbage collection reachability and + # "everything that used X" are plain joins rather than a json_each + # over every payload in the database. + db["part_attachments"].create( + { + "part_id": int, + "attachment_id": str, + "order": int, + }, + pk=("part_id", "attachment_id"), + foreign_keys=( + ("part_id", "parts", "id"), + ("attachment_id", "attachments", "id"), + ), + ) + + db["part_fragments"].create( + { + "part_id": int, + "fragment_id": int, + "order": int, + }, + pk=("part_id", "fragment_id", "order"), + foreign_keys=( + ("part_id", "parts", "id"), + ("fragment_id", "fragments", "id"), + ), + ) + db["part_fragments"].create_index(["fragment_id"]) + + # The only mutable rows in the schema. A fork is a second thread + # pointing at a message that already exists. + db["threads"].create( + { + "id": str, + "name": str, + "tip_message_hash": str, + "forked_from": str, + "datetime_utc": str, + }, + pk="id", + foreign_keys=( + ("tip_message_hash", "messages", "hash"), + ("forked_from", "threads", "id"), + ), + ) + + # One model call. Self-contained - it does not read anything from the + # older responses table. parent_ and tip_message_hash together + # delimit what this turn contributed, which nothing else records, + # because message rows are shared and cannot carry provenance. + db["turns"].create( + { + "id": str, + "thread_id": str, + "parent_message_hash": str, + "tip_message_hash": str, + "model": str, + "resolved_model": str, + "options_json": str, + "schema_id": str, + "input_tokens": int, + "output_tokens": int, + "token_details": str, + "duration_ms": int, + "datetime_utc": str, + }, + pk="id", + foreign_keys=( + ("thread_id", "threads", "id"), + ("parent_message_hash", "messages", "hash"), + ("tip_message_hash", "messages", "hash"), + ("schema_id", "schemas", "id"), + ), + ) + db["turns"].create_index(["thread_id"]) + + db["turn_tools"].create( + {"turn_id": str, "tool_id": int}, + pk=("turn_id", "tool_id"), + foreign_keys=(("turn_id", "turns", "id"), ("tool_id", "tools", "id")), + ) + + # Provenance: which fragments this call was given. Distinct from + # part_fragments, which says what a message's text is built from. + db["turn_fragments"].create( + { + "turn_id": str, + "fragment_id": int, + "order": int, + "kind": str, # 'prompt' | 'system' + }, + pk=("turn_id", "fragment_id", "kind"), + foreign_keys=( + ("turn_id", "turns", "id"), + ("fragment_id", "fragments", "id"), + ), + ) + db["turn_fragments"].create_index(["fragment_id"]) diff --git a/llm/models.py b/llm/models.py index 1a29f6f1e..129533364 100644 --- a/llm/models.py +++ b/llm/models.py @@ -590,6 +590,7 @@ def _build_full_chain( explicit_messages, system=None, system_fragments=None, + fragments=None, ) -> list[Any]: """Build the full message chain for the next turn. @@ -652,8 +653,15 @@ def _build_full_chain( ) user_parts: list[Any] = [] - if prompt: - user_parts.append(TextPart(text=prompt)) + # Fragments are concatenated into the prompt text before it is + # sent, so they have to be in the chain too - prompt.messages is + # meant to be exactly what the model sees. Matches Prompt.prompt. + prompt_text = "\n".join( + [str(fragment) for fragment in fragments or []] + + ([prompt] if prompt else []) + ) + if prompt_text: + user_parts.append(TextPart(text=prompt_text)) for att in attachments or []: user_parts.append(AttachmentPart(attachment=att)) if user_parts: @@ -695,6 +703,7 @@ def prompt( explicit_messages=messages, system=system, system_fragments=system_fragments, + fragments=fragments, ) return Response( Prompt( @@ -749,6 +758,7 @@ def chain( explicit_messages=messages, system=system, system_fragments=system_fragments, + fragments=fragments, ) return ChainResponse( Prompt( @@ -823,6 +833,7 @@ def chain( explicit_messages=messages, system=system, system_fragments=system_fragments, + fragments=fragments, ) return AsyncChainResponse( Prompt( @@ -874,6 +885,7 @@ def prompt( explicit_messages=messages, system=system, system_fragments=system_fragments, + fragments=fragments, ) return AsyncResponse( Prompt( diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index f9bb9336d..d34fdfc47 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -28,8 +28,10 @@ "messages", "parts", "part_attachments", + "part_fragments", "turns", "turn_tools", + "turn_fragments", "threads", } @@ -306,35 +308,6 @@ def test_unknown_tip_raises(self, store): store.load_chain("b2:does-not-exist") -# ---- storage by reference -------------------------------------------- - - -class TestStorageByReference: - def test_text_matching_a_fragment_is_stored_by_reference(self, store): - content = "a large reusable fragment" - ensure_fragment(store.db, content) - store.ensure_chain([llm.user(content)]) - row = next(iter(store.db["parts"].rows)) - assert row["text"] is None - assert row["fragment_id"] is not None - - def test_fragment_backed_text_still_round_trips(self, store): - content = "a large reusable fragment" - ensure_fragment(store.db, content) - messages = [llm.user(content)] - assert round_trip(store, messages) == messages - - def test_fragment_backed_text_hashes_the_same_as_inline(self, store): - content = "a large reusable fragment" - inline_tip = store.ensure_chain([llm.user(content)]) - ensure_fragment(store.db, content) - by_reference_tip = store.ensure_chain([llm.user(content)]) - # Identity is the resolved text, so where the bytes live makes - # no difference to the hash - and the second write is a no-op. - assert inline_tip == by_reference_tip - assert store.db["messages"].count == 1 - - # ---- dedup ----------------------------------------------------------- @@ -761,3 +734,184 @@ def test_successive_library_turns_extend_the_thread(self, store, mock_model): response.text() response.log_to_db(store.db) assert len(store.thread_messages(conversation.id)) == 4 + + +# ---- storage by reference -------------------------------------------- + + +class TestFragmentReferences: + """The point of fragments is that a novel is stored once and pointed + at from every prompt about it, so the text must not be expanded into + each message that uses it.""" + + NOVEL = "CALL ME ISHMAEL. " * 500 + + def messages_using(self, fragment, question): + # What Prompt.prompt builds: fragments joined to the prompt text. + return [llm.user(f"{fragment}\n{question}")] + + def test_fragment_text_is_not_duplicated_into_the_part(self, store): + ensure_fragment(store.db, self.NOVEL) + store.ensure_chain( + self.messages_using(self.NOVEL, "who is the narrator?"), + fragments=[self.NOVEL], + ) + payload = next(iter(store.db["parts"].rows))["payload"] + assert self.NOVEL not in payload + assert len(payload) < 200 + + def test_many_prompts_about_one_fragment_store_it_once(self, store): + ensure_fragment(store.db, self.NOVEL) + for question in ("who?", "where?", "when?", "why?"): + store.ensure_chain( + self.messages_using(self.NOVEL, question), fragments=[self.NOVEL] + ) + assert store.db["fragments"].count == 1 + assert store.db["parts"].count == 4 + total = sum(len(row["payload"]) for row in store.db["parts"].rows) + assert total < len(self.NOVEL) + + def test_referenced_text_round_trips(self, store): + ensure_fragment(store.db, self.NOVEL) + messages = self.messages_using(self.NOVEL, "who is the narrator?") + tip = store.ensure_chain(messages, fragments=[self.NOVEL]) + assert store.load_chain(tip) == messages + + def test_several_fragments_in_one_part_round_trip(self, store): + one, two = "FIRST FRAGMENT", "SECOND FRAGMENT" + for content in (one, two): + ensure_fragment(store.db, content) + messages = [llm.user(f"{one}\n{two}\ncompare them")] + tip = store.ensure_chain(messages, fragments=[one, two]) + assert store.load_chain(tip) == messages + + def test_part_fragments_records_the_link(self, store): + ensure_fragment(store.db, self.NOVEL) + store.ensure_chain( + self.messages_using(self.NOVEL, "who?"), fragments=[self.NOVEL] + ) + assert store.db["part_fragments"].count == 1 + + def test_messages_using_a_fragment_are_one_join_away(self, store): + ensure_fragment(store.db, self.NOVEL) + for question in ("who?", "where?"): + store.ensure_chain( + self.messages_using(self.NOVEL, question), fragments=[self.NOVEL] + ) + fragment_id = next(iter(store.db["fragments"].rows))["id"] + found = list( + store.db.query( + """ + select distinct parts.message_hash from part_fragments + join parts on parts.id = part_fragments.part_id + where part_fragments.fragment_id = ? + """, + [fragment_id], + ) + ) + assert len(found) == 2 + + def test_unknown_fragments_are_stored_inline(self, store): + messages = [llm.user("just some text")] + tip = store.ensure_chain(messages) + assert store.db["part_fragments"].count == 0 + assert store.load_chain(tip) == messages + + def test_hashes_do_not_depend_on_where_the_bytes_live(self, store): + # Identity is the resolved content, so storing by reference must + # produce exactly the hash that storing inline would. + messages = self.messages_using(self.NOVEL, "who?") + inline = store.ensure_chain(messages) + ensure_fragment(store.db, self.NOVEL) + by_reference = store.ensure_chain(messages, fragments=[self.NOVEL]) + assert inline == by_reference + + +# ---- verification ---------------------------------------------------- + + +class TestVerify: + """Reads resolve references, so a reconstruction bug would produce a + chain that differs from what was hashed - and would do it silently. + Re-hashing every stored message catches the whole class at once.""" + + def test_a_fresh_store_verifies(self, store): + assert store.verify() == [] + + def test_every_kind_of_part_verifies(self, store): + novel = "CALL ME ISHMAEL. " * 100 + store.ensure_chain( + [ + llm.system("be brief"), + llm.user( + f"{novel}\nwho is the narrator?", + Attachment(type="image/png", content=b"bytes"), + ), + llm.assistant( + ReasoningPart(text="thinking", provider_metadata={"a": 1}), + ReasoningPart(redacted=True), + ToolCallPart(name="s", arguments={"q": 1}, tool_call_id="tc1"), + TextPart(text="answer"), + ), + llm.tool_message( + ToolResultPart( + name="s", + output="out", + tool_call_id="tc1", + exception="ValueError: x", + attachments=[Attachment(type="image/png", content=b"one")], + ) + ), + ], + fragments=[novel], + ) + assert store.verify() == [] + + def test_a_corrupted_payload_is_caught(self, store): + tip = store.ensure_chain([llm.user("Hi")]) + with store.db.conn: + store.db.execute( + "update parts set payload = ?", ['{"type":"text","text":"tampered"}'] + ) + assert store.verify() == [tip] + + def test_a_missing_fragment_is_caught(self, store): + novel = "CALL ME ISHMAEL. " * 100 + tip = store.ensure_chain([llm.user(f"{novel}\nwho?")], fragments=[novel]) + with store.db.conn: + store.db.execute("delete from fragments") + assert store.verify() == [tip] + + +class TestFragmentsEndToEnd: + + def test_a_conversation_chain_includes_fragment_text(self, mock_model): + # prompt.messages is meant to be exactly what the model sees, and + # what the model sees has the fragments concatenated in. + conversation = mock_model.conversation() + mock_model.enqueue(["ok"]) + response = conversation.prompt("question", fragments=["FRAGMENT-BODY"]) + response.text() + assert response.prompt.messages[-1].parts[0].text == response.prompt.prompt + + def test_the_cli_stores_a_fragment_by_reference(self, user_path, tmpdir): + novel = "CALL ME ISHMAEL. " * 3000 + path = tmpdir / "novel.txt" + path.write_text(novel, "utf-8") + for question in ("who?", "where?", "when?"): + run("-m", "echo", question, "-f", str(path)) + + db = sqlite_utils.Database(str(user_path / "logs.db")) + assert db["fragments"].count == 1 + assert db["part_fragments"].count >= 3 + # Every question re-sends the novel; it must be stored once. + user_payloads = sum( + len(row["payload"]) + for row in db.query( + "select payload from parts join messages" + " on messages.hash = parts.message_hash" + " where messages.role = 'user'" + ) + ) + assert user_payloads < len(novel) + assert LogStore(db).verify() == [] From 58dfe234c31b0e545fd21955cfc0b8cebbb86d2f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 11:48:32 -0700 Subject: [PATCH 127/258] response.execute_tool_call() method Closes #1560 --- docs/plugins/advanced-model-plugins.md | 32 +++++++++ llm/models.py | 60 +++++++++++++--- tests/test_tools.py | 96 ++++++++++++++++++++++++++ 3 files changed, 178 insertions(+), 10 deletions(-) diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index 45de8854a..40de87137 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -380,6 +380,38 @@ response.add_tool_call( ) ``` +### Provider-managed local tool calls + +Some provider SDKs orchestrate the tool loop themselves: they call a local callback with generated arguments, wait for that callback to return a result, and then continue the same model response. Plugins for those providers can delegate the actual invocation to LLM using `response.execute_tool_call()`: + +```python +tool_result = response.execute_tool_call( + llm.ToolCall( + tool_call_id=tool_id, + name=tool_name, + arguments=parsed_args, + ) +) +return tool_result.output +``` + +Async model plugins should use the awaitable equivalent method on `AsyncResponse`: + +```python +tool_result = await response.execute_tool_call( + llm.ToolCall( + tool_call_id=tool_id, + name=tool_name, + arguments=parsed_args, + ) +) +return tool_result.output +``` + +This uses LLM's normal tool executor, including toolbox preparation, error handling, `llm_tool_call` injection and the `before_call` and `after_call` lifecycle callbacks. If `tool_call_id` is omitted LLM generates one. + +Do **not** also call `response.add_tool_call()` for a provider-managed call. That would cause the normal chain loop to execute the same tool a second time after the provider has already received its result. + ### Server-side tool calls For tools the API executes internally, set `server_executed=True` on the events. Anthropic web search is an example: the API returns a `server_tool_use` block for the search request, followed by a `web_search_tool_result` block containing the result payload. diff --git a/llm/models.py b/llm/models.py index ff58d6366..d7679ea64 100644 --- a/llm/models.py +++ b/llm/models.py @@ -336,6 +336,16 @@ class ToolCall: tool_call_id: str | None = None +def _ensure_tool_call_id(tool_call: ToolCall) -> ToolCall: + # Generate a tool call ID if one has not yet been specified + if tool_call.tool_call_id is not None: + return tool_call + return dataclasses.replace( + tool_call, + tool_call_id=f"tc_{str(monotonic_ulid()).lower()}", + ) + + @dataclass class ToolResult: "The result of executing a tool call." @@ -697,6 +707,8 @@ def prompt( stream, conversation=self, key=key, + before_call=self.before_call, + after_call=self.after_call, ) def chain( @@ -876,6 +888,8 @@ def prompt( stream, conversation=self, key=key, + before_call=self.before_call, + after_call=self.after_call, ) def to_sync_conversation(self): @@ -941,6 +955,8 @@ def __init__( stream: bool, conversation: _BaseConversation | None = None, key: str | None = None, + before_call: BeforeCallSync | BeforeCallAsync | None = None, + after_call: AfterCallSync | AfterCallAsync | None = None, ): self.id = str(monotonic_ulid()).lower() self.prompt = prompt @@ -948,6 +964,8 @@ def __init__( self.model = model self.stream = stream self._key = key + self.before_call = before_call + self.after_call = after_call self._chunks: list[str] = [] # Every StreamEvent ever yielded by execute(), in order. Plain # str yields are wrapped as text events (with part_index resolved @@ -1272,16 +1290,7 @@ def _build_parts(self) -> list[Any]: return parts def add_tool_call(self, tool_call: ToolCall): - if tool_call.tool_call_id is None: - # Guarantee every locally-executable tool call has a unique id. - # Some providers never supply one, which otherwise forces every - # consumer correlating calls with results (or keying external - # state on a call) to invent fallback matching schemes. - tool_call = dataclasses.replace( - tool_call, - tool_call_id=f"tc_{str(monotonic_ulid()).lower()}", - ) - self._tool_calls.append(tool_call) + self._tool_calls.append(_ensure_tool_call_id(tool_call)) def set_usage( self, @@ -1944,6 +1953,15 @@ def execute_tool_calls( tool_results.append(tool_result_obj) return tool_results + def execute_tool_call(self, tool_call: ToolCall) -> ToolResult: + "Utility method for manually executing a tool call with callbacks" + tool_call = _ensure_tool_call_id(tool_call) + return self.execute_tool_calls( + before_call=cast(BeforeCallSync | None, self.before_call), + after_call=cast(AfterCallSync | None, self.after_call), + tool_calls_list=[tool_call], + )[0] + def tool_calls(self) -> list[ToolCall]: "Return the list of tool calls made during this response." self._force() @@ -2411,6 +2429,16 @@ async def run_async(tc=tc, tool=tool, idx=idx): return results + async def execute_tool_call(self, tool_call: ToolCall) -> ToolResult: + "Asynchronous counterpart to :meth:`Response.execute_tool_call`." + tool_call = _ensure_tool_call_id(tool_call) + results = await self.execute_tool_calls( + before_call=cast(BeforeCallAsync | None, self.before_call), + after_call=cast(AfterCallAsync | None, self.after_call), + tool_calls_list=[tool_call], + ) + return results[0] + def __aiter__(self): self._start = time.monotonic() self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) @@ -2827,6 +2855,8 @@ def responses(self) -> Iterator[Response]: self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) # Resume: a history ending in unresolved tool calls means a # previous run stopped (paused or crashed) before executing @@ -2846,6 +2876,8 @@ def responses(self) -> Iterator[Response]: self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) current_response: Response | None = initial_response while current_response: @@ -2892,6 +2924,8 @@ def responses(self) -> Iterator[Response]: stream=self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) else: current_response = None @@ -2924,6 +2958,8 @@ async def responses(self) -> AsyncIterator[AsyncResponse]: self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) # Resume: see ChainResponse.responses() - execute trailing # unresolved tool calls before the first provider call. This @@ -2941,6 +2977,8 @@ async def responses(self) -> AsyncIterator[AsyncResponse]: self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) current_response: AsyncResponse | None = initial_response while current_response: @@ -2984,6 +3022,8 @@ async def responses(self) -> AsyncIterator[AsyncResponse]: stream=self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) else: current_response = None diff --git a/tests/test_tools.py b/tests/test_tools.py index e312a0d7a..93c95ba39 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -483,6 +483,102 @@ async def after(*args): assert len(after_collected) == 2 +def test_provider_managed_tool_execution_uses_response_context(): + events = [] + implementation_call = {} + + def lookup(value: str, llm_tool_call: llm.ToolCall) -> dict: + implementation_call["tool_call"] = llm_tool_call + return {"value": value.upper()} + + def before(tool, tool_call): + events.append(("before", tool.name, tool_call.tool_call_id)) + + def after(tool, tool_call, tool_result): + events.append( + ( + "after", + tool.name, + tool_call.tool_call_id, + tool_result.output, + ) + ) + + class ProviderManagedToolModel(llm.Model): + model_id = "provider-managed-tool" + supports_tools = True + + def execute(self, prompt, stream, response, conversation=None): + tool_result = response.execute_tool_call( + llm.ToolCall(name="lookup", arguments={"value": prompt.prompt}) + ) + yield tool_result.output + + chain = ProviderManagedToolModel().chain( + "pelican", + tools=[lookup], + before_call=before, + after_call=after, + ) + + assert chain.text() == '{"value": "PELICAN"}' + assert len(chain._responses) == 1 + tool_call = implementation_call["tool_call"] + assert tool_call.tool_call_id.startswith("tc_") + assert events == [ + ("before", "lookup", tool_call.tool_call_id), + ("after", "lookup", tool_call.tool_call_id, '{"value": "PELICAN"}'), + ] + + +@pytest.mark.asyncio +async def test_async_provider_managed_tool_execution_uses_response_context(): + events = [] + + async def lookup(value: str) -> str: + await asyncio.sleep(0) + return value.upper() + + async def before(tool, tool_call): + events.append(("before", tool.name, tool_call.tool_call_id)) + + async def after(tool, tool_call, tool_result): + events.append( + ( + "after", + tool.name, + tool_call.tool_call_id, + tool_result.output, + ) + ) + + class AsyncProviderManagedToolModel(llm.AsyncModel): + model_id = "async-provider-managed-tool" + supports_tools = True + + async def execute(self, prompt, stream, response, conversation=None): + tool_result = await response.execute_tool_call( + llm.ToolCall(name="lookup", arguments={"value": prompt.prompt}) + ) + yield tool_result.output + + chain = AsyncProviderManagedToolModel().chain( + "puffin", + tools=[lookup], + before_call=before, + after_call=after, + ) + + assert await chain.text() == "PUFFIN" + assert len(chain._responses) == 1 + assert events[0][0:2] == ("before", "lookup") + assert events[0][2].startswith("tc_") + assert events == [ + ("before", "lookup", events[0][2]), + ("after", "lookup", events[0][2], "PUFFIN"), + ] + + ERROR_FUNCTION = """ def trigger_error(msg: str): raise Exception(msg) From 8e005ac8a79b202b7aac0c235f8394907fd7c692 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 13:10:39 -0700 Subject: [PATCH 128/258] Carry stream events through to_sync_response Async responses lost their reasoning. to_sync_response() copied _chunks and _tool_calls but not _stream_events, so the converted response fell back to assembling a bare TextPart from the accumulated text - dropping ReasoningParts, redacted markers and every part's provider_metadata. The CLI converts before logging, so this applied to every async response. Found by running llm --async against llm-anthropic and llm-gemini: the same prompt logged an Anthropic reasoning signature when run sync and nothing at all when run async. Co-Authored-By: Claude Opus 5 --- llm/models.py | 5 ++++ tests/test_logs_store.py | 51 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/llm/models.py b/llm/models.py index 129533364..b44f35a96 100644 --- a/llm/models.py +++ b/llm/models.py @@ -2633,6 +2633,11 @@ async def to_sync_response(self) -> Response: response._prompt_json = self._prompt_json response.response_json = self.response_json response._tool_calls = list(self._tool_calls) + # Without these the sync response falls back to assembling a bare + # TextPart from _chunks, so reasoning, redacted markers and every + # part's provider_metadata are lost. The CLI converts before + # logging, so that loss would apply to every async response. + response._stream_events = list(self._stream_events) response.attachments = list(self.attachments) response.resolved_model = self.resolved_model return response diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index d34fdfc47..4d751c55f 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -915,3 +915,54 @@ def test_the_cli_stores_a_fragment_by_reference(self, user_path, tmpdir): ) assert user_payloads < len(novel) assert LogStore(db).verify() == [] + + +# ---- async ------------------------------------------------------------ + + +class TestAsyncLogging: + """The CLI converts an async response to a sync one before logging, + so anything the conversion drops is dropped from the log.""" + + def enqueue_reasoning(self, model): + model.enqueue( + [ + llm.parts.StreamEvent( + type="reasoning", + chunk="thinking hard", + provider_metadata={"anthropic": {"signature": "SIG"}}, + ), + llm.parts.StreamEvent(type="text", chunk="the answer"), + ] + ) + + async def respond(self, model, prompt="q"): + response = model.prompt(prompt) + await response.text() + return response + + @pytest.mark.asyncio + async def test_to_sync_response_keeps_the_parts(self, async_mock_model): + self.enqueue_reasoning(async_mock_model) + response = await self.respond(async_mock_model) + before = response._messages_now() + after = (await response.to_sync_response())._messages_now() + assert after == before + + @pytest.mark.asyncio + async def test_logging_an_async_response_keeps_reasoning( + self, store, async_mock_model + ): + self.enqueue_reasoning(async_mock_model) + response = await self.respond(async_mock_model) + store.log(await response.to_sync_response()) + + parts = store.load_chain( + next(iter(store.db["turns"].rows))["tip_message_hash"] + )[-1].parts + assert [type(part).__name__ for part in parts] == [ + "ReasoningPart", + "TextPart", + ] + assert parts[0].provider_metadata == {"anthropic": {"signature": "SIG"}} + assert store.verify() == [] From 6ec6a116c12fa1cd1ac482965abd68dbd505995d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 13:18:46 -0700 Subject: [PATCH 129/258] Fix for Windows C: absolute paths being mistaken for a fragment plugin prefix Closes #1563 --- llm/cli.py | 2 +- tests/test_fragments_cli.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/llm/cli.py b/llm/cli.py index ecd8121a7..3b7fb3256 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -168,7 +168,7 @@ def _load_by_alias(fragment: str) -> tuple[str | None, str | None]: resolved.append(Fragment(response.text, fragment)) elif fragment == "-": resolved.append(Fragment(sys.stdin.read(), "-")) - elif has_plugin_prefix(fragment): + elif has_plugin_prefix(fragment) and not pathlib.Path(fragment).exists(): prefix, rest = fragment.split(":", 1) loaders = get_fragment_loaders() if prefix not in loaders: diff --git a/tests/test_fragments_cli.py b/tests/test_fragments_cli.py index d52f3179e..66262cc61 100644 --- a/tests/test_fragments_cli.py +++ b/tests/test_fragments_cli.py @@ -1,3 +1,4 @@ +import json import os import textwrap from importlib.metadata import version @@ -128,6 +129,24 @@ def test_fragments_list(user_path): """).strip()) +def test_fragment_absolute_path(user_path, tmp_path): + path = tmp_path / "fragment.txt" + path.write_text("Hello from an absolute path") + + result = CliRunner().invoke( + cli, ["prompt", "-m", "echo", "-f", str(path)], catch_exceptions=False + ) + + assert result.exit_code == 0 + assert json.loads(result.output) == { + "prompt": "Hello from an absolute path", + "system": "", + "attachments": [], + "stream": True, + "previous": [], + } + + @mock.patch.dict(os.environ, {"OPENAI_API_KEY": "X"}) def test_fragment_url_user_agent(mocked_openai_chat, user_path): mocked_openai_chat.add_response( From 237f047859ac53f9b7335a7d10a2bf1b1feb8f97 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 13:18:46 -0700 Subject: [PATCH 130/258] Fix for Windows C: absolute paths being mistaken for a fragment plugin prefix Closes #1563 --- llm/cli.py | 2 +- tests/test_fragments_cli.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/llm/cli.py b/llm/cli.py index 43ba9ae8b..0fc3a09eb 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -169,7 +169,7 @@ def _load_by_alias(fragment: str) -> tuple[str | None, str | None]: resolved.append(Fragment(response.text, fragment)) elif fragment == "-": resolved.append(Fragment(sys.stdin.read(), "-")) - elif has_plugin_prefix(fragment): + elif has_plugin_prefix(fragment) and not pathlib.Path(fragment).exists(): prefix, rest = fragment.split(":", 1) loaders = get_fragment_loaders() if prefix not in loaders: diff --git a/tests/test_fragments_cli.py b/tests/test_fragments_cli.py index d52f3179e..66262cc61 100644 --- a/tests/test_fragments_cli.py +++ b/tests/test_fragments_cli.py @@ -1,3 +1,4 @@ +import json import os import textwrap from importlib.metadata import version @@ -128,6 +129,24 @@ def test_fragments_list(user_path): """).strip()) +def test_fragment_absolute_path(user_path, tmp_path): + path = tmp_path / "fragment.txt" + path.write_text("Hello from an absolute path") + + result = CliRunner().invoke( + cli, ["prompt", "-m", "echo", "-f", str(path)], catch_exceptions=False + ) + + assert result.exit_code == 0 + assert json.loads(result.output) == { + "prompt": "Hello from an absolute path", + "system": "", + "attachments": [], + "stream": True, + "previous": [], + } + + @mock.patch.dict(os.environ, {"OPENAI_API_KEY": "X"}) def test_fragment_url_user_agent(mocked_openai_chat, user_path): mocked_openai_chat.add_response( From 7b9c981fa9589d94e95d52bbfb9d8d0ed42bc891 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 11:48:32 -0700 Subject: [PATCH 131/258] response.execute_tool_call() method Closes #1560 --- docs/plugins/advanced-model-plugins.md | 32 +++++++++ llm/models.py | 60 +++++++++++++--- tests/test_tools.py | 96 ++++++++++++++++++++++++++ 3 files changed, 178 insertions(+), 10 deletions(-) diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index 45de8854a..40de87137 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -380,6 +380,38 @@ response.add_tool_call( ) ``` +### Provider-managed local tool calls + +Some provider SDKs orchestrate the tool loop themselves: they call a local callback with generated arguments, wait for that callback to return a result, and then continue the same model response. Plugins for those providers can delegate the actual invocation to LLM using `response.execute_tool_call()`: + +```python +tool_result = response.execute_tool_call( + llm.ToolCall( + tool_call_id=tool_id, + name=tool_name, + arguments=parsed_args, + ) +) +return tool_result.output +``` + +Async model plugins should use the awaitable equivalent method on `AsyncResponse`: + +```python +tool_result = await response.execute_tool_call( + llm.ToolCall( + tool_call_id=tool_id, + name=tool_name, + arguments=parsed_args, + ) +) +return tool_result.output +``` + +This uses LLM's normal tool executor, including toolbox preparation, error handling, `llm_tool_call` injection and the `before_call` and `after_call` lifecycle callbacks. If `tool_call_id` is omitted LLM generates one. + +Do **not** also call `response.add_tool_call()` for a provider-managed call. That would cause the normal chain loop to execute the same tool a second time after the provider has already received its result. + ### Server-side tool calls For tools the API executes internally, set `server_executed=True` on the events. Anthropic web search is an example: the API returns a `server_tool_use` block for the search request, followed by a `web_search_tool_result` block containing the result payload. diff --git a/llm/models.py b/llm/models.py index b44f35a96..baca25454 100644 --- a/llm/models.py +++ b/llm/models.py @@ -336,6 +336,16 @@ class ToolCall: tool_call_id: str | None = None +def _ensure_tool_call_id(tool_call: ToolCall) -> ToolCall: + # Generate a tool call ID if one has not yet been specified + if tool_call.tool_call_id is not None: + return tool_call + return dataclasses.replace( + tool_call, + tool_call_id=f"tc_{str(monotonic_ulid()).lower()}", + ) + + @dataclass class ToolResult: "The result of executing a tool call." @@ -724,6 +734,8 @@ def prompt( stream, conversation=self, key=key, + before_call=self.before_call, + after_call=self.after_call, ) def chain( @@ -906,6 +918,8 @@ def prompt( stream, conversation=self, key=key, + before_call=self.before_call, + after_call=self.after_call, ) def to_sync_conversation(self): @@ -971,6 +985,8 @@ def __init__( stream: bool, conversation: _BaseConversation | None = None, key: str | None = None, + before_call: BeforeCallSync | BeforeCallAsync | None = None, + after_call: AfterCallSync | AfterCallAsync | None = None, ): self.id = str(monotonic_ulid()).lower() self.prompt = prompt @@ -978,6 +994,8 @@ def __init__( self.model = model self.stream = stream self._key = key + self.before_call = before_call + self.after_call = after_call self._chunks: list[str] = [] # Every StreamEvent ever yielded by execute(), in order. Plain # str yields are wrapped as text events (with part_index resolved @@ -1302,16 +1320,7 @@ def _build_parts(self) -> list[Any]: return parts def add_tool_call(self, tool_call: ToolCall): - if tool_call.tool_call_id is None: - # Guarantee every locally-executable tool call has a unique id. - # Some providers never supply one, which otherwise forces every - # consumer correlating calls with results (or keying external - # state on a call) to invent fallback matching schemes. - tool_call = dataclasses.replace( - tool_call, - tool_call_id=f"tc_{str(monotonic_ulid()).lower()}", - ) - self._tool_calls.append(tool_call) + self._tool_calls.append(_ensure_tool_call_id(tool_call)) def set_usage( self, @@ -1986,6 +1995,15 @@ def execute_tool_calls( tool_results.append(tool_result_obj) return tool_results + def execute_tool_call(self, tool_call: ToolCall) -> ToolResult: + "Utility method for manually executing a tool call with callbacks" + tool_call = _ensure_tool_call_id(tool_call) + return self.execute_tool_calls( + before_call=cast(BeforeCallSync | None, self.before_call), + after_call=cast(AfterCallSync | None, self.after_call), + tool_calls_list=[tool_call], + )[0] + def tool_calls(self) -> list[ToolCall]: "Return the list of tool calls made during this response." self._force() @@ -2453,6 +2471,16 @@ async def run_async(tc=tc, tool=tool, idx=idx): return results + async def execute_tool_call(self, tool_call: ToolCall) -> ToolResult: + "Asynchronous counterpart to :meth:`Response.execute_tool_call`." + tool_call = _ensure_tool_call_id(tool_call) + results = await self.execute_tool_calls( + before_call=cast(BeforeCallAsync | None, self.before_call), + after_call=cast(AfterCallAsync | None, self.after_call), + tool_calls_list=[tool_call], + ) + return results[0] + def __aiter__(self): self._start = time.monotonic() self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) @@ -2874,6 +2902,8 @@ def responses(self) -> Iterator[Response]: self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) # Resume: a history ending in unresolved tool calls means a # previous run stopped (paused or crashed) before executing @@ -2893,6 +2923,8 @@ def responses(self) -> Iterator[Response]: self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) current_response: Response | None = initial_response while current_response: @@ -2939,6 +2971,8 @@ def responses(self) -> Iterator[Response]: stream=self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) else: current_response = None @@ -2971,6 +3005,8 @@ async def responses(self) -> AsyncIterator[AsyncResponse]: self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) # Resume: see ChainResponse.responses() - execute trailing # unresolved tool calls before the first provider call. This @@ -2988,6 +3024,8 @@ async def responses(self) -> AsyncIterator[AsyncResponse]: self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) current_response: AsyncResponse | None = initial_response while current_response: @@ -3031,6 +3069,8 @@ async def responses(self) -> AsyncIterator[AsyncResponse]: stream=self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) else: current_response = None diff --git a/tests/test_tools.py b/tests/test_tools.py index e312a0d7a..93c95ba39 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -483,6 +483,102 @@ async def after(*args): assert len(after_collected) == 2 +def test_provider_managed_tool_execution_uses_response_context(): + events = [] + implementation_call = {} + + def lookup(value: str, llm_tool_call: llm.ToolCall) -> dict: + implementation_call["tool_call"] = llm_tool_call + return {"value": value.upper()} + + def before(tool, tool_call): + events.append(("before", tool.name, tool_call.tool_call_id)) + + def after(tool, tool_call, tool_result): + events.append( + ( + "after", + tool.name, + tool_call.tool_call_id, + tool_result.output, + ) + ) + + class ProviderManagedToolModel(llm.Model): + model_id = "provider-managed-tool" + supports_tools = True + + def execute(self, prompt, stream, response, conversation=None): + tool_result = response.execute_tool_call( + llm.ToolCall(name="lookup", arguments={"value": prompt.prompt}) + ) + yield tool_result.output + + chain = ProviderManagedToolModel().chain( + "pelican", + tools=[lookup], + before_call=before, + after_call=after, + ) + + assert chain.text() == '{"value": "PELICAN"}' + assert len(chain._responses) == 1 + tool_call = implementation_call["tool_call"] + assert tool_call.tool_call_id.startswith("tc_") + assert events == [ + ("before", "lookup", tool_call.tool_call_id), + ("after", "lookup", tool_call.tool_call_id, '{"value": "PELICAN"}'), + ] + + +@pytest.mark.asyncio +async def test_async_provider_managed_tool_execution_uses_response_context(): + events = [] + + async def lookup(value: str) -> str: + await asyncio.sleep(0) + return value.upper() + + async def before(tool, tool_call): + events.append(("before", tool.name, tool_call.tool_call_id)) + + async def after(tool, tool_call, tool_result): + events.append( + ( + "after", + tool.name, + tool_call.tool_call_id, + tool_result.output, + ) + ) + + class AsyncProviderManagedToolModel(llm.AsyncModel): + model_id = "async-provider-managed-tool" + supports_tools = True + + async def execute(self, prompt, stream, response, conversation=None): + tool_result = await response.execute_tool_call( + llm.ToolCall(name="lookup", arguments={"value": prompt.prompt}) + ) + yield tool_result.output + + chain = AsyncProviderManagedToolModel().chain( + "puffin", + tools=[lookup], + before_call=before, + after_call=after, + ) + + assert await chain.text() == "PUFFIN" + assert len(chain._responses) == 1 + assert events[0][0:2] == ("before", "lookup") + assert events[0][2].startswith("tc_") + assert events == [ + ("before", "lookup", events[0][2]), + ("after", "lookup", events[0][2], "PUFFIN"), + ] + + ERROR_FUNCTION = """ def trigger_error(msg: str): raise Exception(msg) From 626f8e2f220c2a90f3fb97719621e19e7fafafae Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 14:28:41 -0700 Subject: [PATCH 132/258] Refactor terminal chat loop for reuse --- llm/cli.py | 165 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 103 insertions(+), 62 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index 0fc3a09eb..c6b3482b6 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -126,6 +126,92 @@ async def display_async_stream_events(events, *, show_reasoning=True): click.echo(click.style(event.chunk, dim=True), nl=False, err=True) +def _run_chat( + model_label, + prompt_callback, + *, + db=None, + initial_fragments=None, + initial_attachments=None, + transform_prompt=None, + after_response=None, + show_reasoning=True, +): + """Run the terminal chat loop shared by managed and transient models.""" + click.echo(f"Chatting with {model_label}") + click.echo("Type 'exit' or 'quit' to exit") + click.echo("Type '!multi' to enter multiple lines, then '!end' to finish") + click.echo("Type '!edit' to open your default editor and modify the prompt") + if db is not None: + click.echo( + "Type '!fragment [ ...]' to insert one or more fragments" + ) + + argument_fragments = list(initial_fragments or []) + argument_attachments = list(initial_attachments or []) + in_multi = False + accumulated = [] + accumulated_fragments = [] + accumulated_attachments = [] + end_token = "!end" + + while True: + prompt = click.prompt("", prompt_suffix="> " if not in_multi else "") + fragments = [] + attachments = [] + if argument_fragments: + fragments += argument_fragments + # Fragments from command options are added to the first message only. + argument_fragments = [] + if argument_attachments: + attachments = argument_attachments + argument_attachments = [] + if prompt.strip().startswith("!multi"): + in_multi = True + bits = prompt.strip().split() + if len(bits) > 1: + end_token = "!end {}".format(" ".join(bits[1:])) + continue + if prompt.strip() == "!edit": + edited_prompt = click.edit() + if edited_prompt is None: + click.echo("Editor closed without saving.", err=True) + continue + prompt = edited_prompt.strip() + if db is not None and prompt.strip().startswith("!fragment "): + prompt, fragments, attachments = process_fragments_in_chat(db, prompt) + + if in_multi: + if prompt.strip() == end_token: + prompt = "\n".join(accumulated) + fragments = accumulated_fragments + attachments = accumulated_attachments + in_multi = False + accumulated = [] + accumulated_fragments = [] + accumulated_attachments = [] + else: + if prompt: + accumulated.append(prompt) + accumulated_fragments += fragments + accumulated_attachments += attachments + continue + + if transform_prompt is not None: + prompt = transform_prompt(prompt) + if prompt.strip() in ("exit", "quit"): + break + + response = prompt_callback(prompt, fragments, attachments) + display_stream_events( + response.stream_events(), + show_reasoning=show_reasoning, + ) + if after_response is not None: + after_response(response) + print() + + def validate_fragment_alias(ctx, param, value): if not re.match(r"^[a-zA-Z0-9_-]+$", value): raise click.BadParameter("Fragment alias must be alphanumeric") @@ -1219,60 +1305,8 @@ def chat( except FragmentNotFound as ex: raise click.ClickException(str(ex)) - click.echo(f"Chatting with {model.model_id}") - click.echo("Type 'exit' or 'quit' to exit") - click.echo("Type '!multi' to enter multiple lines, then '!end' to finish") - click.echo("Type '!edit' to open your default editor and modify the prompt") - click.echo( - "Type '!fragment [ ...]' to insert one or more fragments" - ) - in_multi = False - - accumulated = [] - accumulated_fragments = [] - accumulated_attachments = [] - end_token = "!end" - while True: - prompt = click.prompt("", prompt_suffix="> " if not in_multi else "") - fragments = [] - attachments = [] - if argument_fragments: - fragments += argument_fragments - # fragments from --fragments will get added to the first message only - argument_fragments = [] - if argument_attachments: - attachments = argument_attachments - argument_attachments = [] - if prompt.strip().startswith("!multi"): - in_multi = True - bits = prompt.strip().split() - if len(bits) > 1: - end_token = "!end {}".format(" ".join(bits[1:])) - continue - if prompt.strip() == "!edit": - edited_prompt = click.edit() - if edited_prompt is None: - click.echo("Editor closed without saving.", err=True) - continue - prompt = edited_prompt.strip() - if prompt.strip().startswith("!fragment "): - prompt, fragments, attachments = process_fragments_in_chat(db, prompt) - - if in_multi: - if prompt.strip() == end_token: - prompt = "\n".join(accumulated) - fragments = accumulated_fragments - attachments = accumulated_attachments - in_multi = False - accumulated = [] - accumulated_fragments = [] - accumulated_attachments = [] - else: - if prompt: - accumulated.append(prompt) - accumulated_fragments += fragments - accumulated_attachments += attachments - continue + def transform_chat_prompt(prompt): + nonlocal system if template_obj: try: # Mirror prompt() logic: only pass input if template uses it @@ -1288,9 +1322,10 @@ def chat( prompt = f"{template_prompt}\n{prompt}" else: prompt = template_prompt - if prompt.strip() in ("exit", "quit"): - break + return prompt + def execute_chat_prompt(prompt, fragments, attachments): + nonlocal system, argument_system_fragments response = conversation.chain( prompt, fragments=fragments, @@ -1303,12 +1338,18 @@ def chat( # System prompt and system fragments only sent for the first message system = None argument_system_fragments = [] - display_stream_events( - response.stream_events(), - show_reasoning=not hide_reasoning, - ) - response.log_to_db(db) - print() + return response + + _run_chat( + model.model_id, + execute_chat_prompt, + db=db, + initial_fragments=argument_fragments, + initial_attachments=argument_attachments, + transform_prompt=transform_chat_prompt, + after_response=lambda response: response.log_to_db(db), + show_reasoning=not hide_reasoning, + ) def load_conversation( From b1c79146e693a2e63ad363456ff35ad8f111652a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 14:28:49 -0700 Subject: [PATCH 133/258] Add transient OpenAI-compatible endpoint command --- README.md | 2 + docs/help.md | 32 ++- docs/other-models.md | 44 +++++ llm/default_plugins/openai_models.py | 152 ++++++++++++++- tests/test_openai_endpoint.py | 280 +++++++++++++++++++++++++++ 5 files changed, 505 insertions(+), 5 deletions(-) create mode 100644 tests/test_openai_endpoint.py diff --git a/README.md b/README.md index d3da2444e..4ccf271ad 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,8 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Other models](https://llm.datasette.io/en/stable/other-models.html) * [Installing and using a local model](https://llm.datasette.io/en/stable/other-models.html#installing-and-using-a-local-model) * [OpenAI-compatible models](https://llm.datasette.io/en/stable/other-models.html#openai-compatible-models) + * [Run against an endpoint without configuring it](https://llm.datasette.io/en/stable/other-models.html#run-against-an-endpoint-without-configuring-it) + * [Configure an OpenAI-compatible model](https://llm.datasette.io/en/stable/other-models.html#configure-an-openai-compatible-model) * [Extra HTTP headers](https://llm.datasette.io/en/stable/other-models.html#extra-http-headers) * [Tools](https://llm.datasette.io/en/stable/tools.html) * [How tools work](https://llm.datasette.io/en/stable/tools.html#how-tools-work) diff --git a/docs/help.md b/docs/help.md index 0503832bf..9085f25cf 100644 --- a/docs/help.md +++ b/docs/help.md @@ -80,7 +80,7 @@ Commands: keys Manage stored API keys for different models logs Tools for exploring logged prompts and responses models Manage available models - openai Commands for working directly with the OpenAI API + openai Commands for working with OpenAI and OpenAI-compatible APIs plugins List installed plugins schemas Manage stored schemas similar Return top N similar IDs from a collection using cosine... @@ -1064,13 +1064,39 @@ Options: ``` Usage: llm openai [OPTIONS] COMMAND [ARGS]... - Commands for working directly with the OpenAI API + Commands for working with OpenAI and OpenAI-compatible APIs Options: -h, --help Show this message and exit. Commands: - models List models available to you from the OpenAI API + endpoint Run against an OpenAI-compatible endpoint without logging. + models List models available to you from the OpenAI API +``` + +(help-openai-endpoint)= +#### llm openai endpoint --help +``` +Usage: llm openai endpoint [OPTIONS] URL [PROMPT] + + Run against an OpenAI-compatible endpoint without logging. + + If PROMPT is provided, execute it once. If PROMPT is omitted in an interactive + terminal, start a chat. Piped stdin is treated as a one-off prompt unless + --chat is specified. + +Options: + -m, --model TEXT Model ID to send to the endpoint [required] + -s, --system TEXT System prompt to use + -o, --option ... key/value options for the model + --key TEXT API key or stored key alias to send + -H, --header ... Additional HTTP header + --responses Use the Responses API instead of Chat Completions + --chat Start an interactive chat, even when stdin is not + a terminal + --no-stream Do not stream output + -R, --hide-reasoning Hide reasoning output + -h, --help Show this message and exit. ``` (help-openai-models)= diff --git a/docs/other-models.md b/docs/other-models.md index cd65ce38d..f58f367a8 100644 --- a/docs/other-models.md +++ b/docs/other-models.md @@ -28,6 +28,50 @@ Check the {ref}`plugin directory ` for the latest list of avai Projects such as [LocalAI](https://localai.io/) offer a REST API that imitates the OpenAI API but can be used to run other models, including models that can be installed on your own machine. These can be added using the same configuration mechanism. +### Run against an endpoint without configuring it + +Use `llm openai endpoint` to run a prompt directly against an +OpenAI-compatible base URL: + +```bash +llm openai endpoint https://example.com/v1 \ + -m model-id \ + "What is the capital of France?" +``` + +This command does not register the model and does not log the prompt or +response. It also does not send your configured OpenAI API key to the endpoint. +Use `--key` to explicitly provide a key or the alias of a key saved using +`llm keys set`: + +```bash +llm openai endpoint https://example.com/v1 \ + -m model-id \ + --key custom-endpoint \ + "What is the capital of France?" +``` + +Omit the prompt to start an interactive chat: + +```bash +llm openai endpoint https://example.com/v1 -m model-id +``` + +Piped stdin is treated as a one-off prompt. Use `--chat` to explicitly start +an interactive chat when stdin is not a terminal. + +The command uses the Chat Completions API by default. Add `--responses` for an +endpoint that implements the Responses API: + +```bash +llm openai endpoint https://example.com/v1 \ + -m model-id \ + --responses \ + "What is the capital of France?" +``` + +### Configure an OpenAI-compatible model + The `model_id` is the name LLM will use for the model. The `model_name` is the name which needs to be passed to the API - this might differ from the `model_id`, especially if the `model_id` could potentially clash with other installed models. The `api_base` key can be used to point the OpenAI client library at a different API endpoint. diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 30583150f..cc929ddac 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1,6 +1,7 @@ import datetime import json import os +import sys from collections.abc import AsyncGenerator, Iterable, Iterator from enum import Enum from typing import Any, cast @@ -9,7 +10,7 @@ import httpx import openai import yaml -from pydantic import Field, create_model, field_validator +from pydantic import Field, ValidationError, create_model, field_validator import llm from llm import ( @@ -403,7 +404,154 @@ def embed_batch(self, items: Iterable[str | bytes]) -> Iterator[list[float]]: def register_commands(cli): @cli.group(name="openai") def openai_(): - "Commands for working directly with the OpenAI API" + "Commands for working with OpenAI and OpenAI-compatible APIs" + + @openai_.command() + @click.argument("url") + @click.argument("prompt", required=False) + @click.option( + "model_id", + "-m", + "--model", + required=True, + help="Model ID to send to the endpoint", + ) + @click.option("-s", "--system", help="System prompt to use") + @click.option( + "options", + "-o", + "--option", + type=(str, str), + multiple=True, + help="key/value options for the model", + ) + @click.option("--key", help="API key or stored key alias to send") + @click.option( + "headers", + "-H", + "--header", + type=(str, str), + multiple=True, + help="Additional HTTP header", + ) + @click.option( + "use_responses", + "--responses", + is_flag=True, + help="Use the Responses API instead of Chat Completions", + ) + @click.option( + "force_chat", + "--chat", + is_flag=True, + help="Start an interactive chat, even when stdin is not a terminal", + ) + @click.option("--no-stream", is_flag=True, help="Do not stream output") + @click.option("-R", "--hide-reasoning", is_flag=True, help="Hide reasoning output") + def endpoint( + url, + prompt, + model_id, + system, + options, + key, + headers, + use_responses, + force_chat, + no_stream, + hide_reasoning, + ): + """ + Run against an OpenAI-compatible endpoint without logging. + + If PROMPT is provided, execute it once. If PROMPT is omitted in an + interactive terminal, start a chat. Piped stdin is treated as a + one-off prompt unless --chat is specified. + """ + from llm.cli import _run_chat, display_stream_events, render_errors + + if force_chat and prompt is not None: + raise click.ClickException("--chat cannot be used with a prompt") + + model_class = Responses if use_responses else Chat + model = model_class( + model_id=model_id, + model_name=model_id, + api_base=url, + headers=dict(headers), + ) + + # A configured api_base never receives the user's default OpenAI key. + # Match that safety property here: only send credentials when --key + # was explicitly provided for this invocation. + if not key: + model.needs_key = None + + try: + validated_options = { + option_name: option_value + for option_name, option_value in model.Options(**dict(options)) + if option_value is not None + } + except ValidationError as ex: + raise click.ClickException(render_errors(ex.errors())) + + prompt_kwargs = { + "options": validated_options, + "stream": not no_stream, + "hide_reasoning": hide_reasoning, + } + if key: + prompt_kwargs["key"] = key + + is_chat = force_chat or (prompt is None and sys.stdin.isatty()) + try: + if is_chat: + conversation = model.conversation() + + def execute_chat_prompt(chat_prompt, _fragments, _attachments): + nonlocal system + response = conversation.prompt( + chat_prompt, + system=system, + **prompt_kwargs, + ) + system = None + return response + + _run_chat( + f"{model_id} at {url}", + execute_chat_prompt, + show_reasoning=not hide_reasoning, + ) + return + + if not sys.stdin.isatty(): + stdin_prompt = sys.stdin.read() + if stdin_prompt: + prompt = " ".join( + part for part in (stdin_prompt, prompt) if part is not None + ) + if prompt is None: + raise click.ClickException( + "A prompt is required when stdin is not interactive" + ) + response = model.prompt(prompt, system=system, **prompt_kwargs) + display_stream_events( + response.stream_events(), + show_reasoning=not hide_reasoning, + ) + click.echo() + except (click.Abort, click.ClickException): + raise + except (ValueError, NotImplementedError) as ex: + raise click.ClickException(str(ex)) + except Exception as ex: + if getattr(sys, "_called_from_test", False) or os.environ.get( + "LLM_RAISE_ERRORS" + ): + raise + raise click.ClickException(str(ex)) @openai_.command() @click.option("json_", "--json", is_flag=True, help="Output as JSON") diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py new file mode 100644 index 000000000..ade08fac7 --- /dev/null +++ b/tests/test_openai_endpoint.py @@ -0,0 +1,280 @@ +import json + +from click.testing import CliRunner +from pytest_httpx import IteratorStream + +from llm.cli import cli + + +def _add_chat_response(httpx_mock, url, text): + httpx_mock.add_response( + method="POST", + url=f"{url}/chat/completions", + json={ + "id": "chatcmpl_test", + "object": "chat.completion", + "created": 1, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": text}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 3, + "completion_tokens": 2, + "total_tokens": 5, + }, + }, + headers={"Content-Type": "application/json"}, + ) + + +def _responses_payload(text): + return { + "id": "resp_test", + "object": "response", + "created_at": 1, + "model": "test-model", + "output": [ + { + "type": "message", + "id": "msg_test", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": text, + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 3, + "output_tokens": 2, + "total_tokens": 5, + }, + "status": "completed", + } + + +def _chat_stream_events(): + for delta, finish_reason in ( + ({"role": "assistant", "content": ""}, None), + ({"content": "Hello"}, None), + ({"content": " streamed"}, None), + ({}, "stop"), + ): + yield "data: {}\n\n".format( + json.dumps( + { + "id": "chatcmpl_stream_test", + "object": "chat.completion.chunk", + "created": 1, + "model": "test-model", + "choices": [ + { + "index": 0, + "delta": delta, + "finish_reason": finish_reason, + } + ], + } + ) + ).encode("utf-8") + yield b"data: [DONE]\n\n" + + +def test_endpoint_chat_completions_does_not_log_or_leak_default_key( + httpx_mock, user_path, monkeypatch +): + base_url = "https://example.test/v1" + _add_chat_response(httpx_mock, base_url, "Hello from the endpoint") + monkeypatch.setenv("OPENAI_API_KEY", "real-default-openai-key") + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Hello", + "-m", + "test-model", + "--no-stream", + "-H", + "X-Test", + "one", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "Hello from the endpoint\n" + assert not (user_path / "logs.db").exists() + + request = httpx_mock.get_requests()[0] + assert request.headers["Authorization"] == "Bearer DUMMY_KEY" + assert request.headers["X-Test"] == "one" + assert json.loads(request.content) == { + "messages": [{"role": "user", "content": "Hello"}], + "model": "test-model", + "stream": False, + } + + +def test_endpoint_streams_by_default(httpx_mock, user_path): + base_url = "https://stream.example.test/v1" + httpx_mock.add_response( + method="POST", + url=f"{base_url}/chat/completions", + stream=IteratorStream(_chat_stream_events()), + headers={"Content-Type": "text/event-stream"}, + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Hello", + "-m", + "test-model", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "Hello streamed\n" + assert not (user_path / "logs.db").exists() + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["stream"] is True + assert request_body["stream_options"] == {"include_usage": True} + + +def test_endpoint_uses_explicit_key(httpx_mock, user_path): + base_url = "https://example.test/v1" + _add_chat_response(httpx_mock, base_url, "Authenticated") + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Hello", + "-m", + "test-model", + "--no-stream", + "--key", + "endpoint-key", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert httpx_mock.get_requests()[0].headers["Authorization"] == ( + "Bearer endpoint-key" + ) + assert not (user_path / "logs.db").exists() + + +def test_endpoint_responses_api(httpx_mock, user_path): + base_url = "https://responses.example.test/v1" + httpx_mock.add_response( + method="POST", + url=f"{base_url}/responses", + json=_responses_payload("Hello from Responses"), + headers={"Content-Type": "application/json"}, + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Hello", + "-m", + "test-model", + "--responses", + "--no-stream", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "Hello from Responses\n" + assert not (user_path / "logs.db").exists() + assert json.loads(httpx_mock.get_requests()[0].content) == { + "input": [{"role": "user", "content": "Hello"}], + "model": "test-model", + "store": False, + "stream": False, + } + + +def test_endpoint_reads_one_off_prompt_from_stdin(httpx_mock, user_path): + base_url = "https://stdin.example.test/v1" + _add_chat_response(httpx_mock, base_url, "From stdin") + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "-m", + "test-model", + "--no-stream", + ], + input="Hello from stdin", + catch_exceptions=False, + ) + + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["messages"] == [{"role": "user", "content": "Hello from stdin"}] + assert not (user_path / "logs.db").exists() + + +def test_endpoint_interactive_chat_preserves_history(httpx_mock, user_path): + base_url = "https://chat.example.test/v1" + _add_chat_response(httpx_mock, base_url, "First answer") + _add_chat_response(httpx_mock, base_url, "Second answer") + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "-m", + "test-model", + "--chat", + "--no-stream", + "--system", + "Be brief", + ], + input="First question\nSecond question\nquit\n", + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert "First answer" in result.output + assert "Second answer" in result.output + assert not (user_path / "logs.db").exists() + + requests = httpx_mock.get_requests() + assert len(requests) == 2 + assert json.loads(requests[1].content)["messages"] == [ + {"role": "system", "content": "Be brief"}, + {"role": "user", "content": "First question"}, + {"role": "assistant", "content": "First answer"}, + {"role": "user", "content": "Second question"}, + ] From fe1d296d3abfb4367222445fe05271a743fdffd9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 14:34:13 -0700 Subject: [PATCH 134/258] Add attachments to endpoint command --- docs/help.md | 27 ++++--- docs/other-models.md | 14 ++++ llm/default_plugins/openai_models.py | 35 ++++++++- tests/test_openai_endpoint.py | 105 ++++++++++++++++++++++++++- 4 files changed, 167 insertions(+), 14 deletions(-) diff --git a/docs/help.md b/docs/help.md index 9085f25cf..5aa7c982a 100644 --- a/docs/help.md +++ b/docs/help.md @@ -1086,17 +1086,22 @@ Usage: llm openai endpoint [OPTIONS] URL [PROMPT] --chat is specified. Options: - -m, --model TEXT Model ID to send to the endpoint [required] - -s, --system TEXT System prompt to use - -o, --option ... key/value options for the model - --key TEXT API key or stored key alias to send - -H, --header ... Additional HTTP header - --responses Use the Responses API instead of Chat Completions - --chat Start an interactive chat, even when stdin is not - a terminal - --no-stream Do not stream output - -R, --hide-reasoning Hide reasoning output - -h, --help Show this message and exit. + -m, --model TEXT Model ID to send to the endpoint [required] + -s, --system TEXT System prompt to use + -o, --option ... key/value options for the model + -a, --attachment ATTACHMENT Attachment path or URL or - + --at, --attachment-type ... + Attachment with explicit mimetype, + --at image.jpg image/jpeg + --key TEXT API key or stored key alias to send + -H, --header ... Additional HTTP header + --responses Use the Responses API instead of Chat + Completions + --chat Start an interactive chat, even when stdin is + not a terminal + --no-stream Do not stream output + -R, --hide-reasoning Hide reasoning output + -h, --help Show this message and exit. ``` (help-openai-models)= diff --git a/docs/other-models.md b/docs/other-models.md index f58f367a8..dd9bfc7f1 100644 --- a/docs/other-models.md +++ b/docs/other-models.md @@ -60,6 +60,20 @@ llm openai endpoint https://example.com/v1 -m model-id Piped stdin is treated as a one-off prompt. Use `--chat` to explicitly start an interactive chat when stdin is not a terminal. +Use `-a` or `--attachment` to attach an image or PDF. Chat Completions +endpoints can also receive WAV or MP3 audio attachments: + +```bash +llm openai endpoint https://example.com/v1 \ + -m model-id \ + -a image.jpg \ + "Describe this image" +``` + +Use `--at path-or-url mimetype` when the attachment type cannot be inferred. +Attachments provided when starting an interactive chat are included with the +first message. + The command uses the Chat Completions API by default. Add `--responses` for an endpoint that implements the Responses API: diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index cc929ddac..f42af1f17 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -402,6 +402,8 @@ def embed_batch(self, items: Iterable[str | bytes]) -> Iterator[list[float]]: @hookimpl def register_commands(cli): + from llm.cli import AttachmentType, attachment_types_callback + @cli.group(name="openai") def openai_(): "Commands for working with OpenAI and OpenAI-compatible APIs" @@ -425,6 +427,23 @@ def openai_(): multiple=True, help="key/value options for the model", ) + @click.option( + "attachments", + "-a", + "--attachment", + type=AttachmentType(), + multiple=True, + help="Attachment path or URL or -", + ) + @click.option( + "attachment_types", + "--at", + "--attachment-type", + type=(str, str), + multiple=True, + callback=attachment_types_callback, + help="\b\nAttachment with explicit mimetype,\n--at image.jpg image/jpeg", + ) @click.option("--key", help="API key or stored key alias to send") @click.option( "headers", @@ -454,6 +473,8 @@ def endpoint( model_id, system, options, + attachments, + attachment_types, key, headers, use_responses, @@ -479,6 +500,8 @@ def endpoint( model_name=model_id, api_base=url, headers=dict(headers), + vision=True, + audio=not use_responses, ) # A configured api_base never receives the user's default OpenAI key. @@ -504,16 +527,18 @@ def endpoint( if key: prompt_kwargs["key"] = key + resolved_attachments = [*attachments, *attachment_types] is_chat = force_chat or (prompt is None and sys.stdin.isatty()) try: if is_chat: conversation = model.conversation() - def execute_chat_prompt(chat_prompt, _fragments, _attachments): + def execute_chat_prompt(chat_prompt, _fragments, turn_attachments): nonlocal system response = conversation.prompt( chat_prompt, system=system, + attachments=turn_attachments, **prompt_kwargs, ) system = None @@ -522,6 +547,7 @@ def execute_chat_prompt(chat_prompt, _fragments, _attachments): _run_chat( f"{model_id} at {url}", execute_chat_prompt, + initial_attachments=resolved_attachments, show_reasoning=not hide_reasoning, ) return @@ -536,7 +562,12 @@ def execute_chat_prompt(chat_prompt, _fragments, _attachments): raise click.ClickException( "A prompt is required when stdin is not interactive" ) - response = model.prompt(prompt, system=system, **prompt_kwargs) + response = model.prompt( + prompt, + system=system, + attachments=resolved_attachments, + **prompt_kwargs, + ) display_stream_events( response.stream_events(), show_reasoning=not hide_reasoning, diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py index ade08fac7..030e9dfd2 100644 --- a/tests/test_openai_endpoint.py +++ b/tests/test_openai_endpoint.py @@ -1,3 +1,4 @@ +import base64 import json from click.testing import CliRunner @@ -127,6 +128,50 @@ def test_endpoint_chat_completions_does_not_log_or_leak_default_key( } +def test_endpoint_chat_completions_attachment(httpx_mock, user_path, tmp_path): + base_url = "https://attachments.example.test/v1" + _add_chat_response(httpx_mock, base_url, "A test image") + image_bytes = b"\x89PNG\r\n\x1a\nendpoint attachment" + image_path = tmp_path / "image.png" + image_path.write_bytes(image_bytes) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Describe this", + "-m", + "test-model", + "--no-stream", + "-a", + str(image_path), + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "A test image\n" + assert not (user_path / "logs.db").exists() + assert json.loads(httpx_mock.get_requests()[0].content)["messages"] == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,{}".format( + base64.b64encode(image_bytes).decode("ascii") + ) + }, + }, + ], + } + ] + + def test_endpoint_streams_by_default(httpx_mock, user_path): base_url = "https://stream.example.test/v1" httpx_mock.add_response( @@ -219,6 +264,50 @@ def test_endpoint_responses_api(httpx_mock, user_path): } +def test_endpoint_responses_api_attachment(httpx_mock, user_path): + base_url = "https://responses-attachments.example.test/v1" + httpx_mock.add_response( + method="POST", + url=f"{base_url}/responses", + json=_responses_payload("A remote image"), + headers={"Content-Type": "application/json"}, + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Describe this", + "-m", + "test-model", + "--responses", + "--no-stream", + "--at", + "https://images.example.test/test.jpg", + "image/jpeg", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "A remote image\n" + assert not (user_path / "logs.db").exists() + assert json.loads(httpx_mock.get_requests()[0].content)["input"] == [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Describe this"}, + { + "type": "input_image", + "image_url": "https://images.example.test/test.jpg", + }, + ], + } + ] + + def test_endpoint_reads_one_off_prompt_from_stdin(httpx_mock, user_path): base_url = "https://stdin.example.test/v1" _add_chat_response(httpx_mock, base_url, "From stdin") @@ -260,6 +349,9 @@ def test_endpoint_interactive_chat_preserves_history(httpx_mock, user_path): "--no-stream", "--system", "Be brief", + "--at", + "https://images.example.test/context.jpg", + "image/jpeg", ], input="First question\nSecond question\nquit\n", catch_exceptions=False, @@ -274,7 +366,18 @@ def test_endpoint_interactive_chat_preserves_history(httpx_mock, user_path): assert len(requests) == 2 assert json.loads(requests[1].content)["messages"] == [ {"role": "system", "content": "Be brief"}, - {"role": "user", "content": "First question"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "First question"}, + { + "type": "image_url", + "image_url": { + "url": "https://images.example.test/context.jpg", + }, + }, + ], + }, {"role": "assistant", "content": "First answer"}, {"role": "user", "content": "Second question"}, ] From f22449a87227c121157ff38bdc6e2568f4824f33 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 14:37:19 -0700 Subject: [PATCH 135/258] Add endpoint model listing --- docs/help.md | 7 ++-- docs/other-models.md | 8 +++++ llm/default_plugins/openai_models.py | 26 ++++++++++++--- tests/test_openai_endpoint.py | 50 ++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 7 deletions(-) diff --git a/docs/help.md b/docs/help.md index 5aa7c982a..ca1c49164 100644 --- a/docs/help.md +++ b/docs/help.md @@ -1083,10 +1083,12 @@ Usage: llm openai endpoint [OPTIONS] URL [PROMPT] If PROMPT is provided, execute it once. If PROMPT is omitted in an interactive terminal, start a chat. Piped stdin is treated as a one-off prompt unless - --chat is specified. + --chat is specified. Use --models to list the available model IDs without + running a prompt. Options: - -m, --model TEXT Model ID to send to the endpoint [required] + -m, --model TEXT Model ID to send to the endpoint (required + unless --models) -s, --system TEXT System prompt to use -o, --option ... key/value options for the model -a, --attachment ATTACHMENT Attachment path or URL or - @@ -1099,6 +1101,7 @@ Options: Completions --chat Start an interactive chat, even when stdin is not a terminal + --models List model IDs from the endpoint and exit --no-stream Do not stream output -R, --hide-reasoning Hide reasoning output -h, --help Show this message and exit. diff --git a/docs/other-models.md b/docs/other-models.md index dd9bfc7f1..78f27010b 100644 --- a/docs/other-models.md +++ b/docs/other-models.md @@ -51,6 +51,14 @@ llm openai endpoint https://example.com/v1 \ "What is the capital of France?" ``` +List the model IDs advertised by the endpoint using `--models`. This requests +the `models` resource relative to the base URL, so a base URL ending in `/v1` +will request `/v1/models`: + +```bash +llm openai endpoint https://example.com/v1 --models +``` + Omit the prompt to start an interactive chat: ```bash diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index f42af1f17..6cd729061 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -415,8 +415,7 @@ def openai_(): "model_id", "-m", "--model", - required=True, - help="Model ID to send to the endpoint", + help="Model ID to send to the endpoint (required unless --models)", ) @click.option("-s", "--system", help="System prompt to use") @click.option( @@ -465,6 +464,12 @@ def openai_(): is_flag=True, help="Start an interactive chat, even when stdin is not a terminal", ) + @click.option( + "list_models", + "--models", + is_flag=True, + help="List model IDs from the endpoint and exit", + ) @click.option("--no-stream", is_flag=True, help="Do not stream output") @click.option("-R", "--hide-reasoning", is_flag=True, help="Hide reasoning output") def endpoint( @@ -479,6 +484,7 @@ def endpoint( headers, use_responses, force_chat, + list_models, no_stream, hide_reasoning, ): @@ -487,17 +493,22 @@ def endpoint( If PROMPT is provided, execute it once. If PROMPT is omitted in an interactive terminal, start a chat. Piped stdin is treated as a - one-off prompt unless --chat is specified. + one-off prompt unless --chat is specified. Use --models to list the + available model IDs without running a prompt. """ from llm.cli import _run_chat, display_stream_events, render_errors + if list_models and prompt is not None: + raise click.ClickException("--models cannot be used with a prompt") + if not list_models and not model_id: + raise click.ClickException("--model is required unless --models is used") if force_chat and prompt is not None: raise click.ClickException("--chat cannot be used with a prompt") model_class = Responses if use_responses else Chat model = model_class( - model_id=model_id, - model_name=model_id, + model_id=model_id or "", + model_name=model_id or "", api_base=url, headers=dict(headers), vision=True, @@ -530,6 +541,11 @@ def endpoint( resolved_attachments = [*attachments, *attachment_types] is_chat = force_chat or (prompt is None and sys.stdin.isatty()) try: + if list_models: + for available_model in model.get_client(key).models.list(): + click.echo(available_model.id) + return + if is_chat: conversation = model.conversation() diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py index 030e9dfd2..78b16cbdc 100644 --- a/tests/test_openai_endpoint.py +++ b/tests/test_openai_endpoint.py @@ -229,6 +229,56 @@ def test_endpoint_uses_explicit_key(httpx_mock, user_path): assert not (user_path / "logs.db").exists() +def test_endpoint_lists_models_without_model_or_logging( + httpx_mock, user_path, monkeypatch +): + base_url = "https://models.example.test/v1" + monkeypatch.setenv("OPENAI_API_KEY", "real-default-openai-key") + httpx_mock.add_response( + method="GET", + url=f"{base_url}/models", + json={ + "object": "list", + "data": [ + { + "id": "first-model", + "object": "model", + "created": 1, + "owned_by": "example", + }, + { + "id": "second-model", + "object": "model", + "created": 2, + "owned_by": "example", + }, + ], + }, + headers={"Content-Type": "application/json"}, + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "--models", + "-H", + "X-Test", + "one", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "first-model\nsecond-model\n" + assert not (user_path / "logs.db").exists() + request = httpx_mock.get_requests()[0] + assert request.headers["Authorization"] == "Bearer DUMMY_KEY" + assert request.headers["X-Test"] == "one" + + def test_endpoint_responses_api(httpx_mock, user_path): base_url = "https://responses.example.test/v1" httpx_mock.add_response( From bbe4b1e338d9841ce2aec3a152402e2dabf4937d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 14:45:19 -0700 Subject: [PATCH 136/258] Add templates to endpoint command --- docs/help.md | 6 +- docs/other-models.md | 14 ++++ llm/cli.py | 98 +++++++++++++++------------- llm/default_plugins/openai_models.py | 60 +++++++++++++++-- tests/test_openai_endpoint.py | 72 +++++++++++++++++++- 5 files changed, 194 insertions(+), 56 deletions(-) diff --git a/docs/help.md b/docs/help.md index ca1c49164..b8421e8d5 100644 --- a/docs/help.md +++ b/docs/help.md @@ -1087,9 +1087,11 @@ Usage: llm openai endpoint [OPTIONS] URL [PROMPT] running a prompt. Options: - -m, --model TEXT Model ID to send to the endpoint (required - unless --models) + -m, --model TEXT Model ID (required unless --models or provided + by template) -s, --system TEXT System prompt to use + -t, --template TEXT Template to use + -p, --param ... Parameters for template -o, --option ... key/value options for the model -a, --attachment ATTACHMENT Attachment path or URL or - --at, --attachment-type ... diff --git a/docs/other-models.md b/docs/other-models.md index 78f27010b..cb899bd83 100644 --- a/docs/other-models.md +++ b/docs/other-models.md @@ -82,6 +82,20 @@ Use `--at path-or-url mimetype` when the attachment type cannot be inferred. Attachments provided when starting an interactive chat are included with the first message. +Use `-t` or `--template` to apply an existing LLM template. Template prompts, +system prompts, defaults, model options, model IDs, and attachments are +supported. Pass template variables using `-p` or `--param`: + +```bash +llm openai endpoint https://example.com/v1 \ + -t summarize \ + -p style concise \ + "Text to summarize" +``` + +The `-m` option can be omitted if the template specifies a model. In an +interactive chat the template is applied to each turn. + The command uses the Chat Completions API by default. Add `--responses` for an endpoint that implements the Responses API: diff --git a/llm/cli.py b/llm/cli.py index c6b3482b6..ea96088e1 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -197,10 +197,10 @@ def _run_chat( accumulated_attachments += attachments continue - if transform_prompt is not None: - prompt = transform_prompt(prompt) if prompt.strip() in ("exit", "quit"): break + if transform_prompt is not None: + prompt = transform_prompt(prompt) response = prompt_callback(prompt, fragments, attachments) display_stream_events( @@ -401,6 +401,48 @@ def attachment_types_callback(ctx, param, values) -> list[Attachment]: return collected +def _apply_template(template, prompt, params, system): + """Apply a loaded template to a prompt and system prompt.""" + try: + uses_input = "input" in template.vars() + input_ = prompt if uses_input else "" + template_prompt, template_system = template.evaluate(input_, params) + except Template.MissingVariables as ex: + raise click.ClickException(str(ex)) + if template_system and not system: + system = template_system + if template_prompt: + if prompt and not uses_input: + prompt = f"{template_prompt}\n{prompt}" + else: + prompt = template_prompt + return prompt, system + + +def _merge_template_options(template, options): + """Add template options unless the same option was provided explicitly.""" + merged_options = list(options) + specified_options = dict(merged_options) + for option_name, option_value in (template.options or {}).items(): + if option_name not in specified_options: + merged_options.append((option_name, option_value)) + return merged_options + + +def _merge_template_attachments(template, attachments, attachment_types): + """Resolve and prepend attachments declared by a loaded template.""" + if template.attachments: + attachments = [ + resolve_attachment(value) for value in template.attachments + ] + list(attachments) + if template.attachment_types: + attachment_types = [ + resolve_attachment_with_type(item.value, item.type) + for item in template.attachment_types + ] + list(attachment_types) + return attachments, attachment_types + + def json_validator(object_name): def validator(ctx, param, value): if value is None: @@ -829,41 +871,16 @@ def read_prompt(): tools = [*template_obj.tools, *tools] if template_obj.functions and template_obj._functions_is_trusted: python_tools = [template_obj.functions, *python_tools] - input_ = "" if template_obj.options: - # Make options mutable (they start as a tuple) - options = list(options) - # Load any options, provided they were not set using -o already - specified_options = dict(options) - for option_name, option_value in template_obj.options.items(): - if option_name not in specified_options: - options.append((option_name, option_value)) + options = _merge_template_options(template_obj, options) if "input" in template_obj.vars(): - input_ = read_prompt() - try: - template_prompt, template_system = template_obj.evaluate(input_, params) - if template_prompt: - # Combine with user prompt - if prompt and "input" not in template_obj.vars(): - prompt = template_prompt + "\n" + prompt - else: - prompt = template_prompt - if template_system and not system: - system = template_system - except Template.MissingVariables as ex: - raise click.ClickException(str(ex)) + prompt = read_prompt() + prompt, system = _apply_template(template_obj, prompt, params, system) if model_id is None and template_obj.model: model_id = template_obj.model - # Merge in any attachments - if template_obj.attachments: - attachments = [ - resolve_attachment(a) for a in template_obj.attachments - ] + list(attachments) - if template_obj.attachment_types: - attachment_types = [ - resolve_attachment_with_type(at.value, at.type) - for at in template_obj.attachment_types - ] + list(attachment_types) + attachments, attachment_types = _merge_template_attachments( + template_obj, attachments, attachment_types + ) if extract or extract_last: no_stream = True @@ -1308,20 +1325,7 @@ def chat( def transform_chat_prompt(prompt): nonlocal system if template_obj: - try: - # Mirror prompt() logic: only pass input if template uses it - uses_input = "input" in template_obj.vars() - input_ = prompt if uses_input else "" - template_prompt, template_system = template_obj.evaluate(input_, params) - except Template.MissingVariables as ex: - raise click.ClickException(str(ex)) - if template_system and not system: - system = template_system - if template_prompt: - if prompt and not uses_input: - prompt = f"{template_prompt}\n{prompt}" - else: - prompt = template_prompt + prompt, system = _apply_template(template_obj, prompt, params, system) return prompt def execute_chat_prompt(prompt, fragments, attachments): diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 6cd729061..ecf8b0347 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -415,9 +415,18 @@ def openai_(): "model_id", "-m", "--model", - help="Model ID to send to the endpoint (required unless --models)", + help="Model ID (required unless --models or provided by template)", ) @click.option("-s", "--system", help="System prompt to use") + @click.option("-t", "--template", help="Template to use") + @click.option( + "param", + "-p", + "--param", + multiple=True, + type=(str, str), + help="Parameters for template", + ) @click.option( "options", "-o", @@ -477,6 +486,8 @@ def endpoint( prompt, model_id, system, + template, + param, options, attachments, attachment_types, @@ -496,15 +507,45 @@ def endpoint( one-off prompt unless --chat is specified. Use --models to list the available model IDs without running a prompt. """ - from llm.cli import _run_chat, display_stream_events, render_errors + from llm.cli import ( + AttachmentError, + LoadTemplateError, + _apply_template, + _merge_template_attachments, + _merge_template_options, + _run_chat, + display_stream_events, + load_template, + render_errors, + ) if list_models and prompt is not None: raise click.ClickException("--models cannot be used with a prompt") - if not list_models and not model_id: - raise click.ClickException("--model is required unless --models is used") + if list_models and template: + raise click.ClickException("--models cannot be used with --template") if force_chat and prompt is not None: raise click.ClickException("--chat cannot be used with a prompt") + template_obj = None + params = dict(param) + if template: + try: + template_obj = load_template(template) + attachments, attachment_types = _merge_template_attachments( + template_obj, attachments, attachment_types + ) + except (AttachmentError, LoadTemplateError) as ex: + raise click.ClickException(str(ex)) + if not model_id and template_obj.model: + model_id = template_obj.model + if template_obj.options: + options = _merge_template_options(template_obj, options) + + if not list_models and not model_id: + raise click.ClickException( + "--model is required unless --models or a template model is used" + ) + model_class = Responses if use_responses else Chat model = model_class( model_id=model_id or "", @@ -549,6 +590,14 @@ def endpoint( if is_chat: conversation = model.conversation() + def transform_chat_prompt(chat_prompt): + nonlocal system + if template_obj: + chat_prompt, system = _apply_template( + template_obj, chat_prompt, params, system + ) + return chat_prompt + def execute_chat_prompt(chat_prompt, _fragments, turn_attachments): nonlocal system response = conversation.prompt( @@ -564,6 +613,7 @@ def execute_chat_prompt(chat_prompt, _fragments, turn_attachments): f"{model_id} at {url}", execute_chat_prompt, initial_attachments=resolved_attachments, + transform_prompt=transform_chat_prompt, show_reasoning=not hide_reasoning, ) return @@ -574,6 +624,8 @@ def execute_chat_prompt(chat_prompt, _fragments, turn_attachments): prompt = " ".join( part for part in (stdin_prompt, prompt) if part is not None ) + if template_obj: + prompt, system = _apply_template(template_obj, prompt, params, system) if prompt is None: raise click.ClickException( "A prompt is required when stdin is not interactive" diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py index 78b16cbdc..1e3c70cc8 100644 --- a/tests/test_openai_endpoint.py +++ b/tests/test_openai_endpoint.py @@ -172,6 +172,65 @@ def test_endpoint_chat_completions_attachment(httpx_mock, user_path, tmp_path): ] +def test_endpoint_template(httpx_mock, user_path, templates_path): + base_url = "https://templates.example.test/v1" + _add_chat_response(httpx_mock, base_url, "Template response") + (templates_path / "endpoint.yaml").write_text( + """ +model: template-model +system: You are $persona +prompt: "Question: $input" +options: + temperature: 0.4 +attachment_types: +- type: image/jpeg + value: https://images.example.test/template.jpg +""".strip(), + "utf-8", + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Where?", + "--template", + "endpoint", + "--param", + "persona", + "concise", + "--no-stream", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "Template response\n" + assert not (user_path / "logs.db").exists() + assert json.loads(httpx_mock.get_requests()[0].content) == { + "messages": [ + {"role": "system", "content": "You are concise"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Question: Where?"}, + { + "type": "image_url", + "image_url": { + "url": "https://images.example.test/template.jpg" + }, + }, + ], + }, + ], + "model": "template-model", + "stream": False, + "temperature": 0.4, + } + + def test_endpoint_streams_by_default(httpx_mock, user_path): base_url = "https://stream.example.test/v1" httpx_mock.add_response( @@ -382,10 +441,15 @@ def test_endpoint_reads_one_off_prompt_from_stdin(httpx_mock, user_path): assert not (user_path / "logs.db").exists() -def test_endpoint_interactive_chat_preserves_history(httpx_mock, user_path): +def test_endpoint_interactive_chat_preserves_history( + httpx_mock, user_path, templates_path +): base_url = "https://chat.example.test/v1" _add_chat_response(httpx_mock, base_url, "First answer") _add_chat_response(httpx_mock, base_url, "Second answer") + (templates_path / "endpoint-chat.yaml").write_text( + 'prompt: "Question: $input"\n', "utf-8" + ) result = CliRunner().invoke( cli, @@ -399,6 +463,8 @@ def test_endpoint_interactive_chat_preserves_history(httpx_mock, user_path): "--no-stream", "--system", "Be brief", + "--template", + "endpoint-chat", "--at", "https://images.example.test/context.jpg", "image/jpeg", @@ -419,7 +485,7 @@ def test_endpoint_interactive_chat_preserves_history(httpx_mock, user_path): { "role": "user", "content": [ - {"type": "text", "text": "First question"}, + {"type": "text", "text": "Question: First question"}, { "type": "image_url", "image_url": { @@ -429,5 +495,5 @@ def test_endpoint_interactive_chat_preserves_history(httpx_mock, user_path): ], }, {"role": "assistant", "content": "First answer"}, - {"role": "user", "content": "Second question"}, + {"role": "user", "content": "Question: Second question"}, ] From 85859a0e0380aa4641bfcc1475cd40b4963a854a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 14:54:02 -0700 Subject: [PATCH 137/258] Add tool calling to endpoint command --- docs/help.md | 14 +- docs/other-models.md | 21 ++- llm/cli.py | 186 ++++++++++------------ llm/default_plugins/openai_models.py | 55 +++++-- tests/test_openai_endpoint.py | 228 ++++++++++++++++++++++++++- 5 files changed, 389 insertions(+), 115 deletions(-) diff --git a/docs/help.md b/docs/help.md index b8421e8d5..d10cd9edf 100644 --- a/docs/help.md +++ b/docs/help.md @@ -1082,9 +1082,10 @@ Usage: llm openai endpoint [OPTIONS] URL [PROMPT] Run against an OpenAI-compatible endpoint without logging. If PROMPT is provided, execute it once. If PROMPT is omitted in an interactive - terminal, start a chat. Piped stdin is treated as a one-off prompt unless - --chat is specified. Use --models to list the available model IDs without - running a prompt. + terminal, start a chat unless --template is provided. Templates run once by + default; use --chat to apply one interactively. Piped stdin is treated as a + one-off prompt. Use --models to list the available model IDs without running a + prompt. Options: -m, --model TEXT Model ID (required unless --models or provided @@ -1097,6 +1098,13 @@ Options: --at, --attachment-type ... Attachment with explicit mimetype, --at image.jpg image/jpeg + -T, --tool TEXT Name of a tool to make available to the model + --functions TEXT Python code block or file path defining + functions to register as tools + --td, --tools-debug Show full details of tool executions + --ta, --tools-approve Manually approve every tool execution + --cl, --chain-limit INTEGER How many chained tool responses to allow, + default 5, set 0 for unlimited --key TEXT API key or stored key alias to send -H, --header ... Additional HTTP header --responses Use the Responses API instead of Chat diff --git a/docs/other-models.md b/docs/other-models.md index cb899bd83..5a7f257ef 100644 --- a/docs/other-models.md +++ b/docs/other-models.md @@ -94,7 +94,26 @@ llm openai endpoint https://example.com/v1 \ ``` The `-m` option can be omitted if the template specifies a model. In an -interactive chat the template is applied to each turn. +interactive chat started using `--chat`, the template is applied to each turn. +Without `--chat`, a template that provides its own prompt runs once even when +no prompt argument is supplied. + +Use `-T` or `--tool` to make an installed LLM tool available to the model, or +`--functions` to load Python functions from an inline code block or file: + +```bash +llm openai endpoint https://example.com/v1 \ + -m model-id \ + -T llm_time \ + --functions tools.py \ + "What time is it?" +``` + +Tool calls are executed locally and their results are sent back to the endpoint +until it returns a final answer. `--chain-limit` controls the maximum number of +responses, `--tools-debug` shows tool execution details, and `--tools-approve` +asks for confirmation before each call. Tools and trusted Python functions +declared by local templates are supported too. The command uses the Chat Completions API by default. Add `--responses` for an endpoint that implements the Responses API: diff --git a/llm/cli.py b/llm/cli.py index ea96088e1..643882350 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -443,6 +443,15 @@ def _merge_template_attachments(template, attachments, attachment_types): return attachments, attachment_types +def _merge_template_tools(template, tools, python_tools): + """Prepend trusted tool definitions declared by a loaded template.""" + if template.tools: + tools = [*template.tools, *tools] + if template.functions and template._functions_is_trusted: + python_tools = [template.functions, *python_tools] + return tools, python_tools + + def json_validator(object_name): def validator(ctx, param, value): if value is None: @@ -467,6 +476,54 @@ def schema_option(fn): return fn +def tool_options(fn): + """Add the shared CLI options for selecting and executing tools.""" + decorators = ( + click.option( + "tools", + "-T", + "--tool", + multiple=True, + help="Name of a tool to make available to the model", + ), + click.option( + "python_tools", + "--functions", + multiple=True, + help="Python code block or file path defining functions to register as tools", + ), + click.option( + "tools_debug", + "--td", + "--tools-debug", + is_flag=True, + help="Show full details of tool executions", + envvar="LLM_TOOLS_DEBUG", + ), + click.option( + "tools_approve", + "--ta", + "--tools-approve", + is_flag=True, + help="Manually approve every tool execution", + ), + click.option( + "chain_limit", + "--cl", + "--chain-limit", + type=int, + default=5, + help=( + "How many chained tool responses to allow, " + "default 5, set 0 for unlimited" + ), + ), + ) + for decorator in reversed(decorators): + fn = decorator(fn) + return fn + + @click.group( cls=DefaultGroup, default="prompt", @@ -535,42 +592,7 @@ def cli(): callback=attachment_types_callback, help="\b\nAttachment with explicit mimetype,\n--at image.jpg image/jpeg", ) -@click.option( - "tools", - "-T", - "--tool", - multiple=True, - help="Name of a tool to make available to the model", -) -@click.option( - "python_tools", - "--functions", - help="Python code block or file path defining functions to register as tools", - multiple=True, -) -@click.option( - "tools_debug", - "--td", - "--tools-debug", - is_flag=True, - help="Show full details of tool executions", - envvar="LLM_TOOLS_DEBUG", -) -@click.option( - "tools_approve", - "--ta", - "--tools-approve", - is_flag=True, - help="Manually approve every tool execution", -) -@click.option( - "chain_limit", - "--cl", - "--chain-limit", - type=int, - default=5, - help="How many chained tool responses to allow, default 5, set 0 for unlimited", -) +@tool_options @click.option( "options", "-o", @@ -867,10 +889,7 @@ def read_prompt(): system_fragments = [*template_obj.system_fragments, *system_fragments] if template_obj.schema_object: schema = template_obj.schema_object - if template_obj.tools: - tools = [*template_obj.tools, *tools] - if template_obj.functions and template_obj._functions_is_trusted: - python_tools = [template_obj.functions, *python_tools] + tools, python_tools = _merge_template_tools(template_obj, tools, python_tools) if template_obj.options: options = _merge_template_options(template_obj, options) if "input" in template_obj.vars(): @@ -983,17 +1002,13 @@ def read_prompt(): if conversation: prompt_method = conversation.prompt - tool_implementations = _gather_tools(tools, python_tools) - - if tool_implementations: + tool_kwargs = _tool_chain_kwargs( + tools, python_tools, tools_debug, tools_approve, chain_limit + ) + if tool_kwargs: prompt_method = conversation.chain kwargs["options"] = validated_options - kwargs["chain_limit"] = chain_limit - if tools_debug: - kwargs["after_call"] = _debug_tool_call - if tools_approve: - kwargs["before_call"] = _approve_tool_call - kwargs["tools"] = tool_implementations + kwargs.update(tool_kwargs) else: # Merge in options for the .prompt() methods kwargs.update(validated_options) @@ -1152,42 +1167,7 @@ async def inner(): @click.option("--no-stream", is_flag=True, help="Do not stream output") @click.option("-R", "--hide-reasoning", is_flag=True, help="Hide reasoning output") @click.option("--key", help="API key to use") -@click.option( - "tools", - "-T", - "--tool", - multiple=True, - help="Name of a tool to make available to the model", -) -@click.option( - "python_tools", - "--functions", - help="Python code block or file path defining functions to register as tools", - multiple=True, -) -@click.option( - "tools_debug", - "--td", - "--tools-debug", - is_flag=True, - help="Show full details of tool executions", - envvar="LLM_TOOLS_DEBUG", -) -@click.option( - "tools_approve", - "--ta", - "--tools-approve", - is_flag=True, - help="Manually approve every tool execution", -) -@click.option( - "chain_limit", - "--cl", - "--chain-limit", - type=int, - default=5, - help="How many chained tool responses to allow, default 5, set 0 for unlimited", -) +@tool_options def chat( system, model_id, @@ -1243,10 +1223,7 @@ def chat( raise click.ClickException(str(ex)) if model_id is None and template_obj.model: model_id = template_obj.model - if template_obj.tools: - tools = [*template_obj.tools, *tools] - if template_obj.functions and template_obj._functions_is_trusted: - python_tools = [template_obj.functions, *python_tools] + tools, python_tools = _merge_template_tools(template_obj, tools, python_tools) # Figure out which model we are using if model_id is None: @@ -1268,11 +1245,6 @@ def chat( # Ensure it can see the API key conversation.model = model - if tools_debug: - conversation.after_call = _debug_tool_call - if tools_approve: - conversation.before_call = _approve_tool_call - # Validate options validated_options = get_model_options(model.model_id) if options: @@ -1289,11 +1261,9 @@ def chat( if validated_options: kwargs["options"] = validated_options - tool_functions = _gather_tools(tools, python_tools) - - if tool_functions: - kwargs["chain_limit"] = chain_limit - kwargs["tools"] = tool_functions + kwargs.update( + _tool_chain_kwargs(tools, python_tools, tools_debug, tools_approve, chain_limit) + ) should_stream = model.can_stream and not no_stream if not should_stream: @@ -4242,6 +4212,24 @@ def _gather_tools( return tools +def _tool_chain_kwargs( + tool_specs, python_tools, tools_debug, tools_approve, chain_limit +): + """Build Conversation.chain() keyword arguments for CLI-selected tools.""" + tool_implementations = _gather_tools(tool_specs, python_tools) + if not tool_implementations: + return {} + kwargs = { + "tools": tool_implementations, + "chain_limit": chain_limit, + } + if tools_debug: + kwargs["after_call"] = _debug_tool_call + if tools_approve: + kwargs["before_call"] = _approve_tool_call + return kwargs + + def _get_conversation_tools(conversation, tools): if conversation and not tools and conversation.responses: # Copy plugin tools from first response in conversation diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index ecf8b0347..e83e3798d 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -402,7 +402,7 @@ def embed_batch(self, items: Iterable[str | bytes]) -> Iterator[list[float]]: @hookimpl def register_commands(cli): - from llm.cli import AttachmentType, attachment_types_callback + from llm.cli import AttachmentType, attachment_types_callback, tool_options @cli.group(name="openai") def openai_(): @@ -452,6 +452,7 @@ def openai_(): callback=attachment_types_callback, help="\b\nAttachment with explicit mimetype,\n--at image.jpg image/jpeg", ) + @tool_options @click.option("--key", help="API key or stored key alias to send") @click.option( "headers", @@ -491,6 +492,11 @@ def endpoint( options, attachments, attachment_types, + tools, + python_tools, + tools_debug, + tools_approve, + chain_limit, key, headers, use_responses, @@ -503,8 +509,9 @@ def endpoint( Run against an OpenAI-compatible endpoint without logging. If PROMPT is provided, execute it once. If PROMPT is omitted in an - interactive terminal, start a chat. Piped stdin is treated as a - one-off prompt unless --chat is specified. Use --models to list the + interactive terminal, start a chat unless --template is provided. + Templates run once by default; use --chat to apply one interactively. + Piped stdin is treated as a one-off prompt. Use --models to list the available model IDs without running a prompt. """ from llm.cli import ( @@ -513,7 +520,9 @@ def endpoint( _apply_template, _merge_template_attachments, _merge_template_options, + _merge_template_tools, _run_chat, + _tool_chain_kwargs, display_stream_events, load_template, render_errors, @@ -523,6 +532,8 @@ def endpoint( raise click.ClickException("--models cannot be used with a prompt") if list_models and template: raise click.ClickException("--models cannot be used with --template") + if list_models and (tools or python_tools): + raise click.ClickException("--models cannot be used with tools") if force_chat and prompt is not None: raise click.ClickException("--chat cannot be used with a prompt") @@ -540,6 +551,9 @@ def endpoint( model_id = template_obj.model if template_obj.options: options = _merge_template_options(template_obj, options) + tools, python_tools = _merge_template_tools( + template_obj, tools, python_tools + ) if not list_models and not model_id: raise click.ClickException( @@ -554,6 +568,7 @@ def endpoint( headers=dict(headers), vision=True, audio=not use_responses, + supports_tools=True, ) # A configured api_base never receives the user's default OpenAI key. @@ -579,8 +594,13 @@ def endpoint( if key: prompt_kwargs["key"] = key + tool_kwargs = _tool_chain_kwargs( + tools, python_tools, tools_debug, tools_approve, chain_limit + ) resolved_attachments = [*attachments, *attachment_types] - is_chat = force_chat or (prompt is None and sys.stdin.isatty()) + is_chat = force_chat or ( + prompt is None and template_obj is None and sys.stdin.isatty() + ) try: if list_models: for available_model in model.get_client(key).models.list(): @@ -600,11 +620,15 @@ def transform_chat_prompt(chat_prompt): def execute_chat_prompt(chat_prompt, _fragments, turn_attachments): nonlocal system - response = conversation.prompt( + prompt_method = ( + conversation.chain if tool_kwargs else conversation.prompt + ) + response = prompt_method( chat_prompt, system=system, attachments=turn_attachments, **prompt_kwargs, + **tool_kwargs, ) system = None return response @@ -630,12 +654,21 @@ def execute_chat_prompt(chat_prompt, _fragments, turn_attachments): raise click.ClickException( "A prompt is required when stdin is not interactive" ) - response = model.prompt( - prompt, - system=system, - attachments=resolved_attachments, - **prompt_kwargs, - ) + if tool_kwargs: + response = model.conversation().chain( + prompt, + system=system, + attachments=resolved_attachments, + **prompt_kwargs, + **tool_kwargs, + ) + else: + response = model.prompt( + prompt, + system=system, + attachments=resolved_attachments, + **prompt_kwargs, + ) display_stream_events( response.stream_events(), show_reasoning=not hide_reasoning, diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py index 1e3c70cc8..037403dea 100644 --- a/tests/test_openai_endpoint.py +++ b/tests/test_openai_endpoint.py @@ -33,6 +33,45 @@ def _add_chat_response(httpx_mock, url, text): ) +def _add_chat_tool_call_response(httpx_mock, url, name, arguments): + httpx_mock.add_response( + method="POST", + url=f"{url}/chat/completions", + json={ + "id": "chatcmpl_tool_test", + "object": "chat.completion", + "created": 1, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_test", + "type": "function", + "function": { + "name": name, + "arguments": json.dumps(arguments), + }, + } + ], + }, + "finish_reason": "tool_calls", + } + ], + "usage": { + "prompt_tokens": 3, + "completion_tokens": 2, + "total_tokens": 5, + }, + }, + headers={"Content-Type": "application/json"}, + ) + + def _responses_payload(text): return { "id": "resp_test", @@ -63,6 +102,31 @@ def _responses_payload(text): } +def _responses_tool_call_payload(name, arguments): + return { + "id": "resp_tool_test", + "object": "response", + "created_at": 1, + "model": "test-model", + "output": [ + { + "type": "function_call", + "id": "fc_test", + "call_id": "call_test", + "name": name, + "arguments": json.dumps(arguments), + "status": "completed", + } + ], + "usage": { + "input_tokens": 3, + "output_tokens": 2, + "total_tokens": 5, + }, + "status": "completed", + } + + def _chat_stream_events(): for delta, finish_reason in ( ({"role": "assistant", "content": ""}, None), @@ -231,6 +295,100 @@ def test_endpoint_template(httpx_mock, user_path, templates_path): } +def test_endpoint_static_template_runs_once_on_terminal( + httpx_mock, user_path, templates_path, monkeypatch +): + base_url = "https://terminal-template.example.test/v1" + _add_chat_response(httpx_mock, base_url, "Five pelicans") + (templates_path / "pelican.yaml").write_text( + "prompt: List five pelican names\n", "utf-8" + ) + monkeypatch.setattr("click.testing._NamedTextIOWrapper.isatty", lambda self: True) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "-m", + "test-model", + "-t", + "pelican", + "--no-stream", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "Five pelicans\n" + assert not (user_path / "logs.db").exists() + assert json.loads(httpx_mock.get_requests()[0].content)["messages"] == [ + {"role": "user", "content": "List five pelican names"} + ] + + +def test_endpoint_tools_and_functions(httpx_mock, user_path): + base_url = "https://tools.example.test/v1" + _add_chat_tool_call_response(httpx_mock, base_url, "multiply", {"a": 6, "b": 7}) + _add_chat_response(httpx_mock, base_url, "The answer is 42") + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "What is 6 * 7?", + "-m", + "test-model", + "--no-stream", + "-T", + "llm_version", + "--functions", + ( + "def multiply(a: int, b: int) -> int:\n" + ' "Multiply two numbers."\n' + " return a * b\n" + ), + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "The answer is 42\n" + assert not (user_path / "logs.db").exists() + + requests = [json.loads(request.content) for request in httpx_mock.get_requests()] + assert len(requests) == 2 + assert {tool["function"]["name"] for tool in requests[0]["tools"]} == { + "llm_version", + "multiply", + } + assert requests[1]["messages"] == [ + {"role": "user", "content": "What is 6 * 7?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_test", + "type": "function", + "function": { + "name": "multiply", + "arguments": '{"a": 6, "b": 7}', + }, + } + ], + }, + { + "role": "tool", + "content": "42", + "tool_call_id": "call_test", + }, + ] + + def test_endpoint_streams_by_default(httpx_mock, user_path): base_url = "https://stream.example.test/v1" httpx_mock.add_response( @@ -417,6 +575,64 @@ def test_endpoint_responses_api_attachment(httpx_mock, user_path): ] +def test_endpoint_responses_api_tools(httpx_mock, user_path): + base_url = "https://responses-tools.example.test/v1" + httpx_mock.add_response( + method="POST", + url=f"{base_url}/responses", + json=_responses_tool_call_payload("multiply", {"a": 6, "b": 7}), + headers={"Content-Type": "application/json"}, + ) + httpx_mock.add_response( + method="POST", + url=f"{base_url}/responses", + json=_responses_payload("The answer is 42"), + headers={"Content-Type": "application/json"}, + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "What is 6 * 7?", + "-m", + "test-model", + "--responses", + "--no-stream", + "--functions", + ( + "def multiply(a: int, b: int) -> int:\n" + ' "Multiply two numbers."\n' + " return a * b\n" + ), + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "The answer is 42\n" + assert not (user_path / "logs.db").exists() + + requests = [json.loads(request.content) for request in httpx_mock.get_requests()] + assert requests[0]["tools"][0]["name"] == "multiply" + assert requests[1]["input"] == [ + {"role": "user", "content": "What is 6 * 7?"}, + { + "type": "function_call", + "call_id": "call_test", + "name": "multiply", + "arguments": '{"a": 6, "b": 7}', + }, + { + "type": "function_call_output", + "call_id": "call_test", + "output": "42", + }, + ] + + def test_endpoint_reads_one_off_prompt_from_stdin(httpx_mock, user_path): base_url = "https://stdin.example.test/v1" _add_chat_response(httpx_mock, base_url, "From stdin") @@ -465,6 +681,12 @@ def test_endpoint_interactive_chat_preserves_history( "Be brief", "--template", "endpoint-chat", + "--functions", + ( + "def lookup(value: str) -> str:\n" + ' "Look up a value."\n' + " return value\n" + ), "--at", "https://images.example.test/context.jpg", "image/jpeg", @@ -480,7 +702,11 @@ def test_endpoint_interactive_chat_preserves_history( requests = httpx_mock.get_requests() assert len(requests) == 2 - assert json.loads(requests[1].content)["messages"] == [ + request_bodies = [json.loads(request.content) for request in requests] + assert all( + body["tools"][0]["function"]["name"] == "lookup" for body in request_bodies + ) + assert request_bodies[1]["messages"] == [ {"role": "system", "content": "Be brief"}, { "role": "user", From 5244732ff8bbd42a2837ba79874f2a73b22957c6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 17:11:07 -0700 Subject: [PATCH 138/258] llm prompt --json option (#1566) Outputs a JSON array describing the prompt and response, in the same format as llm logs --json. Also works with --no-log and with logging turned off, in which case the response is logged to a temporary in-memory database to build the JSON. Row fetching/annotation logic used by llm logs --json is now share between the two commands. Claude-Session: https://claude.ai/code/session_015EH5c5qubSwqnjkwiw1zLK --- README.md | 1 + docs/help.md | 2 + docs/usage.md | 43 ++++++ llm/cli.py | 372 +++++++++++++++++++++++++++------------------- tests/test_llm.py | 77 ++++++++++ 5 files changed, 340 insertions(+), 155 deletions(-) diff --git a/README.md b/README.md index d3da2444e..34a935ea4 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [System prompts](https://llm.datasette.io/en/stable/usage.html#system-prompts) * [Tools](https://llm.datasette.io/en/stable/usage.html#tools) * [Extracting fenced code blocks](https://llm.datasette.io/en/stable/usage.html#extracting-fenced-code-blocks) + * [JSON output](https://llm.datasette.io/en/stable/usage.html#json-output) * [Schemas](https://llm.datasette.io/en/stable/usage.html#schemas) * [Fragments](https://llm.datasette.io/en/stable/usage.html#fragments) * [Continuing a conversation](https://llm.datasette.io/en/stable/usage.html#continuing-a-conversation) diff --git a/docs/help.md b/docs/help.md index 0503832bf..d84f6616f 100644 --- a/docs/help.md +++ b/docs/help.md @@ -154,6 +154,8 @@ Options: -u, --usage Show token usage -x, --extract Extract first fenced code block --xl, --extract-last Extract last fenced code block + --json Output the response as JSON, same format as + llm logs --json -h, --help Show this message and exit. ``` diff --git a/docs/usage.md b/docs/usage.md index 576359a1b..6295f343e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -239,6 +239,49 @@ Use `--xl/--extract-last` to return the last fenced code block instead of the fi The entire response including explanatory text is still logged to the database, and can be viewed using `llm logs -c`. +(usage-json-output)= +### JSON output + +Add `--json` to get back a JSON array describing the prompt and the response, in the same format as {ref}`llm logs --json `: + +```bash +llm 'Five names for a pet pelican' --json +``` +```json +[ + { + "id": "01jm8ec74wxsdatyn5pq1fp0s5", + "model": "gpt-4o-mini", + "resolved_model": null, + "prompt": "Five names for a pet pelican", + "system": null, + "prompt_json": null, + "options_json": {}, + "response": "1. Captain Beaky\n...", + "reasoning": null, + "response_json": null, + "conversation_id": "01jm8ec74taftdgj2t4zra9z0j", + "duration_ms": 1560, + "datetime_utc": "2025-02-16T22:34:30.374882+00:00", + "input_tokens": 8, + "output_tokens": 62, + "token_details": null, + "conversation_name": "Five names for a pet pelican", + "conversation_model": "gpt-4o-mini", + "schema_json": null, + "prompt_fragments": [], + "system_fragments": [], + "tools": [], + "tool_calls": [], + "tool_results": [], + "attachments": [] + } +] +``` +The array will contain more than one object if the prompt triggered {ref}`tool calls `, since each round-trip with the model is logged as a separate response. + +This works even if logging is turned off or you use `-n/--no-log` - in that case the JSON is assembled without writing anything to your logs database. + (usage-schemas)= ### Schemas diff --git a/llm/cli.py b/llm/cli.py index 3b7fb3256..3517fd871 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -60,7 +60,7 @@ set_default_model, user_dir, ) -from llm.models import ChainResponse, _BaseConversation +from llm.models import ChainResponse, _BaseChainResponse, _BaseConversation from .migrations import migrate from .plugins import load_plugins, pm @@ -513,6 +513,12 @@ def cli(): is_flag=True, help="Extract last fenced code block", ) +@click.option( + "json_output", + "--json", + is_flag=True, + help="Output the response as JSON, same format as llm logs --json", +) def prompt( prompt, system, @@ -546,6 +552,7 @@ def prompt( usage, extract, extract_last, + json_output, ): """ Execute a prompt @@ -777,7 +784,7 @@ def read_prompt(): resolve_attachment_with_type(at.value, at.type) for at in template_obj.attachment_types ] + list(attachment_types) - if extract or extract_last: + if extract or extract_last or json_output: no_stream = True conversation = None @@ -927,7 +934,8 @@ async def inner(): text = ( extract_fenced_code_block(text, last=extract_last) or text ) - print(text) + if not json_output: + print(text) return response response = asyncio.run(inner()) @@ -951,7 +959,8 @@ async def inner(): text = response.text() if extract or extract_last: text = extract_fenced_code_block(text, last=extract_last) or text - print(text) + if not json_output: + print(text) # List of exceptions that should never be raised in pytest: except (ValueError, NotImplementedError) as ex: raise click.ClickException(str(ex)) @@ -980,12 +989,27 @@ async def inner(): ) # Log responses to the database + log_db = None if (logs_on() or log) and not no_log: + log_db = db + elif json_output: + # --json needs logged rows, so use a temporary in-memory database + log_db = sqlite_utils.Database(memory=True) + migrate(log_db) + + if log_db is not None: # Could be Response, AsyncResponse, ChainResponse, AsyncChainResponse if isinstance(response, AsyncResponse): response = asyncio.run(response.to_sync_response()) # At this point ALL forms should have a log_to_db() method that works: - response.log_to_db(db) + response.log_to_db(log_db) + + if json_output: + if isinstance(response, _BaseChainResponse): + response_ids = [response_.id for response_ in response._responses] + else: + response_ids = [response.id] + click.echo(logs_json_for_response_ids(log_db, response_ids)) @cli.command() @@ -1543,6 +1567,189 @@ def logs_turn_off(): order by prompt_attachments."order" """ +FRAGMENTS_SQL = """ +select + {table}.response_id, + fragments.hash, + fragments.id as fragment_id, + fragments.content, + ( + select json_group_array(fragment_aliases.alias) + from fragment_aliases + where fragment_aliases.fragment_id = fragments.id + ) as aliases +from {table} +join fragments on {table}.fragment_id = fragments.id +where {table}.response_id in ({placeholders}) +order by {table}."order" +""" + +# Tool usage information +TOOLS_SQL = """ +SELECT responses.id, +-- Tools related to this response +COALESCE( + (SELECT json_group_array(json_object( + 'id', t.id, + 'hash', t.hash, + 'name', t.name, + 'description', t.description, + 'input_schema', json(t.input_schema) + )) + FROM tools t + JOIN tool_responses tr ON t.id = tr.tool_id + WHERE tr.response_id = responses.id + ), + '[]' +) AS tools, +-- Tool calls for this response +COALESCE( + (SELECT json_group_array(json_object( + 'id', tc.id, + 'tool_id', tc.tool_id, + 'name', tc.name, + 'arguments', json(tc.arguments), + 'tool_call_id', tc.tool_call_id + )) + FROM tool_calls tc + WHERE tc.response_id = responses.id + ), + '[]' +) AS tool_calls, +-- Tool results for this response +COALESCE( + (SELECT json_group_array(json_object( + 'id', tr.id, + 'tool_id', tr.tool_id, + 'name', tr.name, + 'output', tr.output, + 'tool_call_id', tr.tool_call_id, + 'exception', tr.exception, + 'attachments', COALESCE( + (SELECT json_group_array(json_object( + 'id', a.id, + 'type', a.type, + 'path', a.path, + 'url', a.url, + 'content', a.content + )) + FROM tool_results_attachments tra + JOIN attachments a ON tra.attachment_id = a.id + WHERE tra.tool_result_id = tr.id + ), + '[]' + ) + )) + FROM tool_results tr + WHERE tr.response_id = responses.id + ), + '[]' +) AS tool_results +FROM responses +where id in ({placeholders}) +""" + + +def attachments_by_response_id(db, ids): + "Fetch prompt attachments for these response IDs, grouped by response ID" + attachments_by_id = {} + for attachment in db.query(ATTACHMENTS_SQL.format(",".join("?" * len(ids))), ids): + attachments_by_id.setdefault(attachment["response_id"], []).append(attachment) + return attachments_by_id + + +def annotate_log_rows(db, rows, expand=False, truncate=False): + """ + Modify log rows in place to add fragments and tool information and to + decode (or, if truncate is on, remove) their JSON columns + """ + ids = [row["id"] for row in rows] + + # Fetch any prompt or system prompt fragments + prompt_fragments_by_id = {} + system_fragments_by_id = {} + for table, dictionary in ( + ("prompt_fragments", prompt_fragments_by_id), + ("system_fragments", system_fragments_by_id), + ): + for fragment in db.query( + FRAGMENTS_SQL.format(placeholders=",".join("?" * len(ids)), table=table), + ids, + ): + dictionary.setdefault(fragment["response_id"], []).append(fragment) + + tool_info_by_id = { + row["id"]: { + "tools": json.loads(row["tools"]), + "tool_calls": json.loads(row["tool_calls"]), + "tool_results": json.loads(row["tool_results"]), + } + for row in db.query( + TOOLS_SQL.format(placeholders=",".join("?" * len(ids))), ids + ) + } + + for row in rows: + if truncate: + row["prompt"] = truncate_string(row["prompt"] or "") + row["response"] = truncate_string(row["response"] or "") + # Add prompt and system fragments + for key in ("prompt_fragments", "system_fragments"): + row[key] = [ + { + "hash": fragment["hash"], + "content": ( + fragment["content"] + if expand + else truncate_string(fragment["content"]) + ), + "aliases": json.loads(fragment["aliases"]), + } + for fragment in ( + prompt_fragments_by_id.get(row["id"], []) + if key == "prompt_fragments" + else system_fragments_by_id.get(row["id"], []) + ) + ] + # Either decode or remove all JSON keys + keys = list(row.keys()) + for key in keys: + if key.endswith("_json") and row[key] is not None: + if truncate: + del row[key] + else: + row[key] = json.loads(row[key]) + row.update(tool_info_by_id[row["id"]]) + + +def log_rows_as_json(rows, attachments_by_id): + "Serialize annotated log rows to the JSON used by 'llm logs --json'" + for row in rows: + row["attachments"] = [ + {k: v for k, v in attachment.items() if k != "response_id"} + for attachment in attachments_by_id.get(row["id"], []) + ] + return json.dumps(list(rows), indent=2) + + +def logs_json_for_response_ids(db, ids): + """ + Return the JSON that 'llm logs --json' would output for these response IDs, + in chronological order + """ + if not ids: + return "[]" + placeholders = ",".join("?" * len(ids)) + sql = LOGS_SQL.format( + columns=LOGS_COLUMNS, + extra_where=f" where responses.id in ({placeholders})", + order_by="responses.id", + limit="", + ) + rows = list(db.query(sql, ids)) + annotate_log_rows(db, rows) + return log_rows_as_json(rows, attachments_by_response_id(db, ids)) + @logs.command(name="list") @click.option( @@ -1847,42 +2054,7 @@ def logs_list( if not query and not data: rows.reverse() - # Fetch any attachments ids = [row["id"] for row in rows] - attachments = list(db.query(ATTACHMENTS_SQL.format(",".join("?" * len(ids))), ids)) - attachments_by_id = {} - for attachment in attachments: - attachments_by_id.setdefault(attachment["response_id"], []).append(attachment) - - FRAGMENTS_SQL = """ - select - {table}.response_id, - fragments.hash, - fragments.id as fragment_id, - fragments.content, - ( - select json_group_array(fragment_aliases.alias) - from fragment_aliases - where fragment_aliases.fragment_id = fragments.id - ) as aliases - from {table} - join fragments on {table}.fragment_id = fragments.id - where {table}.response_id in ({placeholders}) - order by {table}."order" - """ - - # Fetch any prompt or system prompt fragments - prompt_fragments_by_id = {} - system_fragments_by_id = {} - for table, dictionary in ( - ("prompt_fragments", prompt_fragments_by_id), - ("system_fragments", system_fragments_by_id), - ): - for fragment in db.query( - FRAGMENTS_SQL.format(placeholders=",".join("?" * len(ids)), table=table), - ids, - ): - dictionary.setdefault(fragment["response_id"], []).append(fragment) if data or data_array or data_key or data_ids: # Special case for --data to output valid JSON @@ -1910,132 +2082,22 @@ def logs_list( click.echo(line) return - # Tool usage information - TOOLS_SQL = """ - SELECT responses.id, - -- Tools related to this response - COALESCE( - (SELECT json_group_array(json_object( - 'id', t.id, - 'hash', t.hash, - 'name', t.name, - 'description', t.description, - 'input_schema', json(t.input_schema) - )) - FROM tools t - JOIN tool_responses tr ON t.id = tr.tool_id - WHERE tr.response_id = responses.id - ), - '[]' - ) AS tools, - -- Tool calls for this response - COALESCE( - (SELECT json_group_array(json_object( - 'id', tc.id, - 'tool_id', tc.tool_id, - 'name', tc.name, - 'arguments', json(tc.arguments), - 'tool_call_id', tc.tool_call_id - )) - FROM tool_calls tc - WHERE tc.response_id = responses.id - ), - '[]' - ) AS tool_calls, - -- Tool results for this response - COALESCE( - (SELECT json_group_array(json_object( - 'id', tr.id, - 'tool_id', tr.tool_id, - 'name', tr.name, - 'output', tr.output, - 'tool_call_id', tr.tool_call_id, - 'exception', tr.exception, - 'attachments', COALESCE( - (SELECT json_group_array(json_object( - 'id', a.id, - 'type', a.type, - 'path', a.path, - 'url', a.url, - 'content', a.content - )) - FROM tool_results_attachments tra - JOIN attachments a ON tra.attachment_id = a.id - WHERE tra.tool_result_id = tr.id - ), - '[]' - ) - )) - FROM tool_results tr - WHERE tr.response_id = responses.id - ), - '[]' - ) AS tool_results - FROM responses - where id in ({placeholders}) - """ - tool_info_by_id = { - row["id"]: { - "tools": json.loads(row["tools"]), - "tool_calls": json.loads(row["tool_calls"]), - "tool_results": json.loads(row["tool_results"]), - } - for row in db.query( - TOOLS_SQL.format(placeholders=",".join("?" * len(ids))), ids - ) - } - - for row in rows: - if truncate: - row["prompt"] = truncate_string(row["prompt"] or "") - row["response"] = truncate_string(row["response"] or "") - # Add prompt and system fragments - for key in ("prompt_fragments", "system_fragments"): - row[key] = [ - { - "hash": fragment["hash"], - "content": ( - fragment["content"] - if expand - else truncate_string(fragment["content"]) - ), - "aliases": json.loads(fragment["aliases"]), - } - for fragment in ( - prompt_fragments_by_id.get(row["id"], []) - if key == "prompt_fragments" - else system_fragments_by_id.get(row["id"], []) - ) - ] - # Either decode or remove all JSON keys - keys = list(row.keys()) - for key in keys: - if key.endswith("_json") and row[key] is not None: - if truncate: - del row[key] - else: - row[key] = json.loads(row[key]) - row.update(tool_info_by_id[row["id"]]) + attachments_by_id = attachments_by_response_id(db, ids) + annotate_log_rows(db, rows, expand=expand, truncate=truncate) output = None if json_output: # Output as JSON if requested - for row in rows: - row["attachments"] = [ - {k: v for k, v in attachment.items() if k != "response_id"} - for attachment in attachments_by_id.get(row["id"], []) - ] - output = json.dumps(list(rows), indent=2) + output = log_rows_as_json(rows, attachments_by_id) elif extract or extract_last: # Extract and return first code block for row in rows: output = extract_fenced_code_block(row["response"], last=extract_last) if output is not None: break - elif response: + elif response and rows: # Just output the last response - if rows: - output = rows[-1]["response"] + output = rows[-1]["response"] if output is not None: click.echo(output) diff --git a/tests/test_llm.py b/tests/test_llm.py index 6293ca232..825ca4f3c 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -1,6 +1,7 @@ import json import os import pathlib +from importlib.metadata import version from unittest import mock import pytest @@ -891,6 +892,82 @@ def test_llm_prompt_continue_with_database( assert sqlite_utils.Database(db_path)["responses"].count == 2 +@pytest.mark.parametrize("async_", (False, True)) +def test_llm_prompt_json(logs_db, async_): + "llm --json should output the same JSON as llm logs --json" + runner = CliRunner() + args = ["-m", "echo", "hello world", "--json"] + if async_: + args.append("--async") + result = runner.invoke(cli, args, catch_exceptions=False) + assert result.exit_code == 0, result.output + rows = json.loads(result.output) + assert len(rows) == 1 + row = rows[0] + assert row["model"] == "echo" + assert row["prompt"] == "hello world" + assert json.loads(row["response"])["prompt"] == "hello world" + assert row["attachments"] == [] + assert row["tools"] == [] + assert row["tool_calls"] == [] + assert row["tool_results"] == [] + # Should be identical to the output of llm logs --json + logs_result = runner.invoke( + cli, ["logs", "-n", "1", "--json"], catch_exceptions=False + ) + assert logs_result.exit_code == 0, logs_result.output + assert json.loads(logs_result.output) == rows + + +@pytest.mark.parametrize("logs_args", (["--no-log"], ["-n"], [])) +def test_llm_prompt_json_without_logging(logs_db, logs_args): + "--json should work even when the response is not logged to the database" + runner = CliRunner() + if not logs_args: + # Turn logging off entirely instead + runner.invoke(cli, ["logs", "off"], catch_exceptions=False) + result = runner.invoke( + cli, ["-m", "echo", "hello world", "--json"] + logs_args, catch_exceptions=False + ) + assert result.exit_code == 0, result.output + rows = json.loads(result.output) + assert len(rows) == 1 + assert rows[0]["prompt"] == "hello world" + # But nothing should have been logged + assert logs_db["responses"].count == 0 + + +def test_llm_prompt_json_with_tools(logs_db): + "Each response in a tool chain should be included in the JSON" + runner = CliRunner() + result = runner.invoke( + cli, + [ + "-m", + "echo", + "-T", + "llm_version", + json.dumps({"tool_calls": [{"name": "llm_version"}]}), + "--json", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output + rows = json.loads(result.output) + assert len(rows) == 2 + assert [tool["name"] for tool in rows[0]["tools"]] == ["llm_version"] + assert [call["name"] for call in rows[0]["tool_calls"]] == ["llm_version"] + assert rows[0]["tool_results"] == [] + assert rows[1]["tool_calls"] == [] + assert [result_["output"] for result_ in rows[1]["tool_results"]] == [ + version("llm") + ] + logs_result = runner.invoke( + cli, ["logs", "-n", "2", "--json"], catch_exceptions=False + ) + assert json.loads(logs_result.output) == rows + + def test_default_exports(): "Check key exports in the llm __all__ list" for name in ("Model", "AsyncModel", "get_model", "get_async_model", "schema_dsl"): From 81683548426e3b9aa9c05b3d37d0cd44caebea02 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 18:21:00 -0700 Subject: [PATCH 139/258] WIP: llm logs reads the content-addressed tables Known broken - 11 failing tests, listed below. Committed so the work is not stranded in a working tree. llm logs now draws its rows from turns/messages/parts rather than from responses, and does not fall back to the older tables at all. A turn's prompt and response are derived by splitting its chain at parent_message_hash, since the input chain is a prefix of the full one. -f filters through turn_fragments, so it matches what the flag has always meant: fragments this call was given. Tests for it empty the responses table first. Written naively, six of eight passed before any of this existed, because the legacy path served them; deleting those rows is what makes a pass mean something. Two real bugs fixed on the way: - Parts were serialised with canonical_json, whose sort_keys is for hashing, not storage. Every stored dict came back alphabetised, so tool call arguments lost the order the model produced them in. The round-trip test could not see it: Python dicts compare equal regardless of key order. - The four sites converting a ToolResult into a ToolResultPart dropped exception and attachments. Same family as the chain dropping fragments. Still not arriving, so a fifth construction path exists that I have not found - see test_tool_errors below. Tests that seed the responses table directly are marked xfail rather than deleted, since whether the legacy read path returns is undecided. 50 of them, applied by fixture - log_path, fragments_fixture and schema_log_path all seed rows that llm logs can no longer see. prompt_json and response_json are gone from llm logs --json: the chain holds the structure and the raw provider payload was dropped as redundant with it. The id on tool calls and results now carries a parts row id rather than a tool_results row id - same role, different number space - so tests normalise it instead of pinning a value. Still failing: - test_llm_default_prompt (9) - every should_log case. Reproducing the same flow by hand shows no difference in any expected key, so something in the test's fixture setup differs and I have not found what. - test_tool_errors (2) - ToolResultPart.exception arrives as None. A baseline run with these changes stashed passes all 112 tests in the three affected files, so every remaining failure belongs to this work and none predate it. Co-Authored-By: Claude Opus 5 --- llm/cli.py | 275 +++++++---------------------------- llm/logs.py | 303 ++++++++++++++++++++++++++++++++++++++- llm/models.py | 22 +++ tests/test_llm.py | 12 +- tests/test_llm_logs.py | 34 +++++ tests/test_logs_store.py | 124 ++++++++++++++++ tests/test_plugins.py | 14 +- 7 files changed, 545 insertions(+), 239 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index 0fc3a09eb..0b7a8e18a 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -62,7 +62,7 @@ ) from llm.models import ChainResponse, _BaseConversation -from .logs import LogStore +from .logs import LogStore, log_row_extras, log_rows from .migrations import migrate from .plugins import load_plugins, pm from .utils import ( @@ -1745,160 +1745,60 @@ def logs_list( # Maybe they uninstalled a model, use the -m option as-is model_id = model - sql = LOGS_SQL - order_by = "responses.id desc" if query: - sql = LOGS_SQL_SEARCH - if not latest: - order_by = "responses_fts.rank desc" - - limit = "" - if count is not None and count > 0: - limit = f" limit {count}" - - sql_format = { - "limit": limit, - "columns": LOGS_COLUMNS, - "extra_where": "", - "order_by": order_by, - } - where_bits = [] - sql_params = { - "model": model_id, - "query": query, - "conversation_id": conversation_id, - "id_gt": id_gt, - "id_gte": id_gte, - } - if model_id: - where_bits.append("responses.model = :model") - if conversation_id: - where_bits.append("responses.conversation_id = :conversation_id") - if id_gt: - where_bits.append("responses.id > :id_gt") - if id_gte: - where_bits.append("responses.id >= :id_gte") - if fragments: - # Resolve the fragments to their hashes - fragment_hashes = [ - fragment.id() for fragment in resolve_fragments(db, fragments) - ] - exists_clauses = [] - - for i, fragment_hash in enumerate(fragment_hashes): - exists_clause = f""" - exists ( - select 1 from prompt_fragments - where prompt_fragments.response_id = responses.id - and prompt_fragments.fragment_id in ( - select fragments.id from fragments - where hash = :f{i} - ) - union - select 1 from system_fragments - where system_fragments.response_id = responses.id - and system_fragments.fragment_id in ( - select fragments.id from fragments - where hash = :f{i} - ) - ) - """ - exists_clauses.append(exists_clause) - sql_params[f"f{i}"] = fragment_hash - - where_bits.append(" and ".join(exists_clauses)) - - if any_tools: - # Any response that involved at least one tool result - where_bits.append(""" - exists ( - select 1 - from tool_results - where - tool_results.response_id = responses.id - ) - """) - if tools: - tools_by_name = get_tools() - # Filter responses by tools (must have ALL of the named tools, including plugin) - tool_clauses = [] - for i, tool_name in enumerate(tools): - try: - plugin_name = tools_by_name[tool_name].plugin - except KeyError: - raise click.ClickException(f"Unknown tool: {tool_name}") - - tool_clauses.append(f""" - exists ( - select 1 - from tool_results - join tools on tools.id = tool_results.tool_id - where tool_results.response_id = responses.id - and tools.name = :tool{i} - and tools.plugin = :plugin{i} - ) - """) - sql_params[f"tool{i}"] = tool_name - sql_params[f"plugin{i}"] = plugin_name - - # AND means “must have all” — use OR instead if you want “any of” - where_bits.append(" and ".join(tool_clauses)) - - schema_id = None - if schema: - schema_id = make_schema_id(schema)[0] - where_bits.append("responses.schema_id = :schema_id") - sql_params["schema_id"] = schema_id - - if where_bits: - where_ = " and " if query else " where " - sql_format["extra_where"] = where_ + " and ".join(where_bits) - - final_sql = sql.format(**sql_format) - rows = list(db.query(final_sql, sql_params)) - - # Reverse the order - we do this because we 'order by id desc limit 3' to get the - # 3 most recent results, but we still want to display them in chronological order - # ... except for searches where we don't do this - if not query and not data: - rows.reverse() + raise click.ClickException( + "-q/--query is not supported against the new log tables yet" + ) - # Fetch any attachments - ids = [row["id"] for row in rows] - attachments = list(db.query(ATTACHMENTS_SQL.format(",".join("?" * len(ids))), ids)) - attachments_by_id = {} - for attachment in attachments: - attachments_by_id.setdefault(attachment["response_id"], []).append(attachment) + fragment_hashes = [fragment.id() for fragment in resolve_fragments(db, fragments)] + + schema_id = make_schema_id(schema)[0] if schema else None + + rows = log_rows( + LogStore(db), + count=count if count and count > 0 else None, + model_id=model_id, + thread_id=conversation_id, + fragment_hashes=fragment_hashes, + tool_names=tools, + any_tools=any_tools, + schema_id=schema_id, + id_gt=id_gt, + id_gte=id_gte, + ) - FRAGMENTS_SQL = """ - select - {table}.response_id, - fragments.hash, - fragments.id as fragment_id, - fragments.content, - ( - select json_group_array(fragment_aliases.alias) - from fragment_aliases - where fragment_aliases.fragment_id = fragments.id - ) as aliases - from {table} - join fragments on {table}.fragment_id = fragments.id - where {table}.response_id in ({placeholders}) - order by {table}."order" - """ + # Newest first out of the query, but read chronologically. + if not data: + rows.reverse() - # Fetch any prompt or system prompt fragments - prompt_fragments_by_id = {} - system_fragments_by_id = {} - for table, dictionary in ( - ("prompt_fragments", prompt_fragments_by_id), - ("system_fragments", system_fragments_by_id), - ): - for fragment in db.query( - FRAGMENTS_SQL.format(placeholders=",".join("?" * len(ids)), table=table), - ids, + # Attachments, fragments and tool info, all from the new tables. + store = LogStore(db) + extras_by_id = {row["id"]: log_row_extras(store, row) for row in rows} + attachments_by_id = { + id: extras["attachments"] for id, extras in extras_by_id.items() + } + prompt_fragments_by_id = { + id: extras["prompt_fragments"] for id, extras in extras_by_id.items() + } + system_fragments_by_id = { + id: extras["system_fragments"] for id, extras in extras_by_id.items() + } + tool_info_by_id = { + id: { + "tools": extras["tools"], + "tool_calls": extras["tool_calls"], + "tool_results": extras["tool_results"], + } + for id, extras in extras_by_id.items() + } + for row in rows: + for internal in ( + "_input_parts", + "_output_parts", + "_parent_message_hash", + "_tip_message_hash", ): - dictionary.setdefault(fragment["response_id"], []).append(fragment) + row.pop(internal, None) if data or data_array or data_key or data_ids: # Special case for --data to output valid JSON @@ -1926,81 +1826,6 @@ def logs_list( click.echo(line) return - # Tool usage information - TOOLS_SQL = """ - SELECT responses.id, - -- Tools related to this response - COALESCE( - (SELECT json_group_array(json_object( - 'id', t.id, - 'hash', t.hash, - 'name', t.name, - 'description', t.description, - 'input_schema', json(t.input_schema) - )) - FROM tools t - JOIN tool_responses tr ON t.id = tr.tool_id - WHERE tr.response_id = responses.id - ), - '[]' - ) AS tools, - -- Tool calls for this response - COALESCE( - (SELECT json_group_array(json_object( - 'id', tc.id, - 'tool_id', tc.tool_id, - 'name', tc.name, - 'arguments', json(tc.arguments), - 'tool_call_id', tc.tool_call_id - )) - FROM tool_calls tc - WHERE tc.response_id = responses.id - ), - '[]' - ) AS tool_calls, - -- Tool results for this response - COALESCE( - (SELECT json_group_array(json_object( - 'id', tr.id, - 'tool_id', tr.tool_id, - 'name', tr.name, - 'output', tr.output, - 'tool_call_id', tr.tool_call_id, - 'exception', tr.exception, - 'attachments', COALESCE( - (SELECT json_group_array(json_object( - 'id', a.id, - 'type', a.type, - 'path', a.path, - 'url', a.url, - 'content', a.content - )) - FROM tool_results_attachments tra - JOIN attachments a ON tra.attachment_id = a.id - WHERE tra.tool_result_id = tr.id - ), - '[]' - ) - )) - FROM tool_results tr - WHERE tr.response_id = responses.id - ), - '[]' - ) AS tool_results - FROM responses - where id in ({placeholders}) - """ - tool_info_by_id = { - row["id"]: { - "tools": json.loads(row["tools"]), - "tool_calls": json.loads(row["tool_calls"]), - "tool_results": json.loads(row["tool_results"]), - } - for row in db.query( - TOOLS_SQL.format(placeholders=",".join("?" * len(ids))), ids - ) - } - for row in rows: if truncate: row["prompt"] = truncate_string(row["prompt"] or "") diff --git a/llm/logs.py b/llm/logs.py index 18331a38e..9cbf955df 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -27,6 +27,8 @@ AttachmentPart, Message, Part, + ReasoningPart, + TextPart, ToolCallPart, ToolResultPart, ) @@ -189,7 +191,11 @@ def _write_part( "position": position, "type": payload["type"], "tool_name": payload.get("name"), - "payload": canonical_json(payload), + # Plain dumps, not canonical_json: sorting keys is + # for hashing. Storage keeps the order the model + # produced, so tool call arguments read back as + # they were written. + "payload": json.dumps(payload), } ) .last_pk @@ -636,3 +642,298 @@ def _load(value: str | None) -> dict | None: def _now() -> str: return str(datetime.datetime.now(datetime.timezone.utc)) + + +# -- llm logs support --------------------------------------------------- +# +# Rows shaped like the ones the older `responses` query produced, so the +# existing rendering in llm.cli works unchanged, but derived entirely +# from the content-addressed tables. + +LOG_ROWS_SQL = """ +select + turns.id, + turns.model, + turns.resolved_model, + turns.options_json, + turns.thread_id as conversation_id, + turns.duration_ms, + turns.datetime_utc, + turns.input_tokens, + turns.output_tokens, + turns.token_details, + turns.parent_message_hash, + turns.tip_message_hash, + threads.name as conversation_name, + turns.model as conversation_model, + schemas.content as schema_json +from turns +left join threads on turns.thread_id = threads.id +left join schemas on turns.schema_id = schemas.id +{where} +order by turns.id desc{limit} +""" + + +def _text_of(parts, kind) -> str: + "Concatenated text of every part of ``kind`` in order." + return "".join(part.text for part in parts if isinstance(part, kind) and part.text) + + +class _LogRowBuilder: + """Turns a turn row into the shape `llm logs` renders. + + A turn's prompt is the last message it was given and its response is + whatever it appended, so both are derived from the parent/tip pair + rather than stored a second time. The chain up to the parent is a + prefix of the chain up to the tip, so splitting them is a matter of + length. + """ + + def __init__(self, store: "LogStore"): + self.store = store + + def build(self, row: dict) -> dict: + inputs = self.store.load_chain(row["parent_message_hash"]) + outputs = self.store.load_chain(row["tip_message_hash"])[len(inputs) :] + + prompt_parts = inputs[-1].parts if inputs else [] + system_parts = ( + inputs[0].parts if inputs and inputs[0].role == "system" else [] + ) + out_parts = [part for message in outputs for part in message.parts] + + built = { + key: row[key] + for key in ( + "id", + "model", + "resolved_model", + "options_json", + "conversation_id", + "duration_ms", + "datetime_utc", + "input_tokens", + "output_tokens", + "token_details", + "conversation_name", + "conversation_model", + "schema_json", + ) + } + built.update( + { + "prompt": _text_of(prompt_parts, TextPart), + # None rather than "" when there was no system + # message, matching what was recorded before. + "system": _text_of(system_parts, TextPart) or None, + "response": _text_of(out_parts, TextPart), + "reasoning": _text_of(out_parts, ReasoningPart) or None, + # Neither is stored any more: the chain holds the + # structure, and the raw provider payload was dropped as + # redundant with it. + "prompt_json": None, + "response_json": None, + "_input_parts": prompt_parts, + "_output_parts": out_parts, + # Internal, stripped before rendering - the enrichment + # needs them to find the parts rows behind these parts. + "_parent_message_hash": row["parent_message_hash"], + "_tip_message_hash": row["tip_message_hash"], + } + ) + return built + + +def log_rows( + store: "LogStore", + *, + count: int | None = None, + model_id: str | None = None, + thread_id: str | None = None, + fragment_hashes=(), + tool_names=(), + any_tools: bool = False, + schema_id: str | None = None, + id_gt: str | None = None, + id_gte: str | None = None, +) -> list[dict]: + """Rows for `llm logs`, newest first, drawn from the new tables. + + Deliberately blind to anything logged before this schema existed - + those conversations have no turns, so they simply do not appear. + """ + where: list[str] = [] + params: dict[str, Any] = {} + + if model_id: + where.append("(turns.model = :model or turns.resolved_model = :model)") + params["model"] = model_id + if thread_id: + where.append("turns.thread_id = :thread_id") + params["thread_id"] = thread_id + if id_gt: + where.append("turns.id > :id_gt") + params["id_gt"] = id_gt + if id_gte: + where.append("turns.id >= :id_gte") + params["id_gte"] = id_gte + if schema_id: + where.append("turns.schema_id = :schema_id") + params["schema_id"] = schema_id + + # Fragments come from turn_fragments - what this call was given - + # rather than from the message text, so it matches what -f means. + for index, fragment_hash in enumerate(fragment_hashes): + key = f"fragment_{index}" + where.append( + f"""turns.id in ( + select turn_fragments.turn_id from turn_fragments + join fragments on fragments.id = turn_fragments.fragment_id + where fragments.hash = :{key} + )""" + ) + params[key] = fragment_hash + + # A turn "used" a tool when a tool result was among its inputs, + # matching what -T has always meant: the result came back and was + # fed to the model. The result sits in the turn's parent message. + if any_tools: + where.append(_tool_result_clause()) + for index, tool_name in enumerate(tool_names): + key = f"tool_{index}" + where.append(_tool_result_clause(f"and parts.tool_name = :{key}")) + params[key] = tool_name + + sql = LOG_ROWS_SQL.format( + where=("where " + " and ".join(where)) if where else "", + limit=f" limit {count}" if count else "", + ) + builder = _LogRowBuilder(store) + return [builder.build(row) for row in store.db.query(sql, params)] + + +def _tool_result_clause(extra: str = "") -> str: + return f"""turns.parent_message_hash in ( + select parts.message_hash from parts + where parts.type = 'tool_result' {extra} + )""" + + +def log_row_extras(store: "LogStore", row: dict) -> dict: + """Attachments, fragments and tool info for one `llm logs` row. + + Attachments and tool calls come from the row's parts, which the row + builder kept hold of. Fragments come from turn_fragments - what the + call was given - so `-f` and the displayed list agree. + """ + attachments = [ + _attachment_summary(part.attachment) + for part in row.get("_input_parts", []) + if isinstance(part, AttachmentPart) and part.attachment is not None + ] + + # parts.id and tools.id stand in for the row ids the old + # tool_calls / tool_results tables exposed, so the JSON shape of + # `llm logs` is unchanged. + call_ids = _part_ids(store, row.get("_tip_message_hash"), "tool_call") + result_ids = _part_ids(store, row.get("_parent_message_hash"), "tool_result") + tool_ids = _tool_ids_by_name(store) + + tool_calls = [ + { + "id": call_ids.get(part.tool_call_id), + "tool_id": tool_ids.get(part.name), + "name": part.name, + "arguments": part.arguments, + "tool_call_id": part.tool_call_id, + } + for part in row.get("_output_parts", []) + if isinstance(part, ToolCallPart) + ] + tool_results = [ + { + "id": result_ids.get(part.tool_call_id), + "tool_id": tool_ids.get(part.name), + "name": part.name, + "output": part.output, + "tool_call_id": part.tool_call_id, + "exception": part.exception, + "attachments": [_attachment_summary(a) for a in part.attachments], + } + for part in row.get("_input_parts", []) + if isinstance(part, ToolResultPart) + ] + + # input_schema is rendered as a dict, so decode it here rather than + # handing the caller the raw JSON text out of the column. + tools = [ + {**tool_row, "input_schema": json.loads(tool_row["input_schema"] or "{}")} + for tool_row in store.db.query( + """ + select tools.id, tools.hash, tools.name, tools.description, + tools.input_schema + from tools join turn_tools on turn_tools.tool_id = tools.id + where turn_tools.turn_id = ? + """, + [row["id"]], + ) + ] + + fragments = {"prompt_fragments": [], "system_fragments": []} + for fragment_row in store.db.query( + """ + select turn_fragments.kind, fragments.hash, fragments.content, + (select json_group_array(fragment_aliases.alias) + from fragment_aliases + where fragment_aliases.fragment_id = fragments.id) as aliases + from turn_fragments + join fragments on fragments.id = turn_fragments.fragment_id + where turn_fragments.turn_id = ? + order by turn_fragments."order" + """, + [row["id"]], + ): + key = f"{fragment_row['kind']}_fragments" + fragments[key].append(dict(fragment_row)) + + return { + "attachments": attachments, + "tools": tools, + "tool_calls": tool_calls, + "tool_results": tool_results, + **fragments, + } + + +def _attachment_summary(attachment) -> dict: + "The attachment shape `llm logs` renders." + content = attachment.content or b"" + return { + "id": attachment.id(), + "type": attachment.resolve_type(), + "path": attachment.path, + "url": attachment.url, + "content": bool(content) or None, + "content_length": len(content) or None, + } + + +def _part_ids(store: "LogStore", message_hash: str | None, type: str) -> dict: + "Map tool_call_id to the parts row id, for one message." + if not message_hash: + return {} + return { + json.loads(part_row["payload"]).get("tool_call_id"): part_row["id"] + for part_row in store.db.query( + "select id, payload from parts where message_hash = ? and type = ?", + [message_hash, type], + ) + } + + +def _tool_ids_by_name(store: "LogStore") -> dict: + return { + tool_row["name"]: tool_row["id"] + for tool_row in store.db.query("select id, name from tools") + } diff --git a/llm/models.py b/llm/models.py index baca25454..6daa69557 100644 --- a/llm/models.py +++ b/llm/models.py @@ -512,6 +512,8 @@ def messages(self): name=tr.name, output=tr.output, tool_call_id=tr.tool_call_id, + exception=_format_tool_exception(tr.exception), + attachments=list(tr.attachments or []), ) for tr in self.tool_results ], @@ -656,6 +658,8 @@ def _build_full_chain( name=tr.name, output=tr.output, tool_call_id=tr.tool_call_id, + exception=_format_tool_exception(tr.exception), + attachments=list(tr.attachments or []), ) for tr in tool_results ], @@ -1809,6 +1813,8 @@ def reply( name=tr.name, output=tr.output, tool_call_id=tr.tool_call_id, + exception=_format_tool_exception(tr.exception), + attachments=list(tr.attachments or []), ) for tr in tool_results ], @@ -2170,6 +2176,8 @@ async def reply( name=tr.name, output=tr.output, tool_call_id=tr.tool_call_id, + exception=_format_tool_exception(tr.exception), + attachments=list(tr.attachments or []), ) for tr in tool_results ], @@ -3500,6 +3508,20 @@ def matches(self, query: str) -> bool: return any(query_lower in alias.lower() for alias in all_strings) + +def _format_tool_exception(exception) -> str | None: + """Render a tool's exception the way it is recorded. + + ToolResult carries the exception object; ToolResultPart carries the + rendered string, so the chain has to convert rather than drop it. + """ + if exception is None: + return None + if isinstance(exception, str): + return exception + return f"{exception.__class__.__name__}: {exception!s}" + + def _conversation_name(text): # Collapse whitespace, including newlines text = re.sub(r"\s+", " ", text) diff --git a/tests/test_llm.py b/tests/test_llm.py index 6293ca232..f1d1afc68 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -136,17 +136,11 @@ def test_llm_default_prompt( "model": "gpt-4o-mini", "prompt": "three names \nfor a pet pelican", "system": None, - "prompt_json": { - "messages": [ - {"role": "user", "content": "three names \nfor a pet pelican"} - ] - }, + # prompt_json and response_json are no longer recorded: the + # message chain holds the structure, and the raw provider + # payload was dropped as redundant with it. "options_json": {}, "response": "Bob, Alice, Eve", - "response_json": { - "model": "gpt-4o-mini", - "choices": [{"message": {"content": {"$": f"r:{row['id']}"}}}], - }, # This doesn't have the \n after three names: "conversation_name": "three names for a pet pelican", "conversation_model": "gpt-4o-mini", diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index fb5c8d020..9fdd9f5fb 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -17,6 +17,22 @@ from llm.migrations import migrate from llm.utils import monotonic_ulid +# These tests write rows straight into the legacy `responses` table and +# then assert on `llm logs` output. `llm logs` now reads the +# content-addressed tables only, so rows that exist nowhere else are +# invisible to it. Kept rather than deleted because whether the legacy +# read path comes back is still undecided. +legacy_rows_only = pytest.mark.xfail( + reason="llm logs no longer reads the legacy responses table", +) + +# -q/--query was backed by responses_fts, which is legacy-only. Full +# text search against the new tables has not been designed yet. +search_not_supported = pytest.mark.xfail( + reason="-q/--query has no implementation against the new log tables", +) + + SINGLE_ID = "5843577700ba729bb14c327b30441885" MULTI_ID = "4860edd987df587d042a9eb2b299ce5c" @@ -90,6 +106,7 @@ def schema_log_path(user_path): id_re = re.compile(r"id: \w+") +@legacy_rows_only @pytest.mark.parametrize("usage", (False, True)) def test_logs_text(log_path, usage): runner = CliRunner() @@ -137,6 +154,8 @@ def test_logs_text(log_path, usage): assert output == expected + +@legacy_rows_only def test_logs_text_with_options(user_path): """Test that ## Options section appears when options_json is set""" log_path = str(user_path / "logs_with_options.db") @@ -173,6 +192,7 @@ def test_logs_text_with_options(user_path): assert "- media_resolution: low" in output +@legacy_rows_only def test_logs_token_usage_details_are_markdown_code(user_path): log_path = str(user_path / "logs_token_details.db") db = sqlite_utils.Database(log_path) @@ -208,6 +228,7 @@ def test_logs_token_usage_details_are_markdown_code(user_path): ) in result.output +@legacy_rows_only @pytest.mark.parametrize("n", (None, 0, 2)) def test_logs_json(n, log_path): "Test that logs command correctly returns requested -n records" @@ -227,6 +248,7 @@ def test_logs_json(n, log_path): assert len(logs) == expected_length +@legacy_rows_only @pytest.mark.parametrize( "args", (["-r"], ["--response"], ["list", "-r"], ["list", "--response"]) ) @@ -238,6 +260,7 @@ def test_logs_response_only(args, log_path): assert result.output == 'response\n```python\nprint("hello word")\n```\n' +@legacy_rows_only @pytest.mark.parametrize( "args", ( @@ -259,6 +282,7 @@ def test_logs_extract_first_code(args, log_path): assert result.output == 'print("hello word")\n\n' +@legacy_rows_only @pytest.mark.parametrize( "args", ( @@ -278,6 +302,7 @@ def test_logs_extract_last_code(args, log_path): assert result.output == 'print("hello word")\n\n' +@legacy_rows_only @pytest.mark.parametrize("arg", ("-s", "--short")) @pytest.mark.parametrize("usage", (None, "-u", "--usage")) def test_logs_short(log_path, arg, usage): @@ -361,6 +386,7 @@ def test_logs_filtered(user_path, model, path_option): assert all(record["model"] == model for record in records) +@search_not_supported @pytest.mark.parametrize( "query,extra_args,expected", ( @@ -403,6 +429,7 @@ def _insert(id, text): assert [record["id"] for record in records] == expected +@legacy_rows_only @pytest.mark.parametrize( "args,expected", ( @@ -451,6 +478,8 @@ def test_logs_schema(schema_log_path, args, expected): assert result.output == expected +@legacy_rows_only +@legacy_rows_only def test_logs_schema_data_ids(schema_log_path): db = sqlite_utils.Database(schema_log_path) ulid = ULID.from_timestamp(time.time() + 100) @@ -641,6 +670,7 @@ def make_response(name, prompt_fragment_ids=None, system_fragment_ids=None): } +@legacy_rows_only @pytest.mark.parametrize( "fragment_refs,expected", ( @@ -746,6 +776,7 @@ def test_logs_fragments(fragments_fixture, fragment_refs, expected): assert reshaped2 == expected +@legacy_rows_only def test_logs_fragments_markdown(fragments_fixture): fragments_log_path = fragments_fixture["path"] fragment_hashes_by_slug = fragments_fixture["fragment_hashes_by_slug"] @@ -928,6 +959,7 @@ def test_logs_fragments_markdown(fragments_fixture): assert output.strip() == expected_output.strip() +@legacy_rows_only @pytest.mark.parametrize("arg", ("-e", "--expand")) def test_expand_fragment_json(fragments_fixture, arg): fragments_log_path = fragments_fixture["path"] @@ -949,6 +981,7 @@ def test_expand_fragment_json(fragments_fixture, arg): assert len(fragment2) > 200 +@legacy_rows_only def test_expand_fragment_markdown(fragments_fixture): fragments_log_path = fragments_fixture["path"] fragment_hashes_by_slug = fragments_fixture["fragment_hashes_by_slug"] @@ -1150,6 +1183,7 @@ def test_log_to_db_persists_empty_reasoning_when_absent(logs_db, mock_model): assert not row.get("reasoning") +@legacy_rows_only def test_logs_markdown_renders_reasoning_heading(user_path): """When a row has reasoning text, `llm logs` renders a `## Reasoning` heading between System and Response.""" diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 4d751c55f..668f7d93b 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -6,6 +6,8 @@ history from a stateless client both write only what is new. """ +import json + import pytest import sqlite_utils from click.testing import CliRunner @@ -966,3 +968,125 @@ async def test_logging_an_async_response_keeps_reasoning( ] assert parts[0].provider_metadata == {"anthropic": {"signature": "SIG"}} assert store.verify() == [] + + +# ---- llm logs against the new tables --------------------------------- + + +def forget_legacy(user_path): + """Empty the table the old `llm logs` reads from. + + Every test below runs this first, so a passing assertion can only + have been served by the content-addressed tables. Without it these + tests pass against the legacy path and prove nothing. + """ + db = sqlite_utils.Database(str(user_path / "logs.db")) + with db.conn: + db.execute("delete from responses") + db.close() + + +class TestLogsCommand: + """`llm logs` reads the content-addressed tables only. Conversations + logged before this schema existed are deliberately not shown yet.""" + + def test_shows_a_logged_prompt(self, user_path): + run("-m", "echo", "hello there") + forget_legacy(user_path) + assert "hello there" in run("logs", "-n", "1").output + + def test_json_output_carries_the_turn(self, user_path): + run("-m", "echo", "hello there") + forget_legacy(user_path) + rows = json.loads(run("logs", "-n", "1", "--json").output) + assert len(rows) == 1 + assert rows[0]["model"] == "echo" + assert rows[0]["prompt"] == "hello there" + assert "hello there" in rows[0]["response"] + + def test_count_limits_results(self, user_path): + for word in ("one", "two", "three"): + run("-m", "echo", word) + forget_legacy(user_path) + assert len(json.loads(run("logs", "-n", "2", "--json").output)) == 2 + + def test_results_are_chronological(self, user_path): + for word in ("one", "two", "three"): + run("-m", "echo", word) + forget_legacy(user_path) + rows = json.loads(run("logs", "-n", "0", "--json").output) + assert [r["prompt"] for r in rows] == ["one", "two", "three"] + + def test_filters_by_model(self, user_path): + run("-m", "echo", "hello") + forget_legacy(user_path) + assert json.loads(run("logs", "-m", "echo", "--json").output) + assert json.loads(run("logs", "-m", "gpt-4o", "--json").output) == [] + + def test_filters_to_a_conversation(self, user_path): + run("-m", "echo", "first") + run("-c", "second") + run("-m", "echo", "unrelated") + forget_legacy(user_path) + db = sqlite_utils.Database(str(user_path / "logs.db")) + thread_id = next( + iter(db.query("select thread_id from turns order by id limit 1")) + )["thread_id"] + rows = json.loads(run("logs", "--cid", thread_id, "--json").output) + assert [r["prompt"] for r in rows] == ["first", "second"] + + def test_filters_by_fragment(self, user_path, tmpdir): + path = tmpdir / "frag.txt" + path.write_text("FRAGMENT BODY", "utf-8") + run("-m", "echo", "with fragment", "-f", str(path)) + run("-m", "echo", "without fragment") + forget_legacy(user_path) + + db = sqlite_utils.Database(str(user_path / "logs.db")) + fragment_hash = next(iter(db["fragments"].rows))["hash"] + rows = json.loads(run("logs", "-f", fragment_hash, "--json").output) + # The stored prompt is the resolved text the model was sent. + assert [r["prompt"] for r in rows] == ["FRAGMENT BODY\nwith fragment"] + + def test_filters_by_tool(self, user_path): + run("-m", "echo", '{"tool_calls": [{"name": "llm_version"}]}', "-T", "llm_version") + run("-m", "echo", "no tools here") + forget_legacy(user_path) + assert len(json.loads(run("logs", "-T", "llm_version", "--json").output)) == 1 + + def test_usage_is_reported(self, user_path): + run("-m", "echo", "hello") + forget_legacy(user_path) + rows = json.loads(run("logs", "--json").output) + assert "input_tokens" in rows[0] + assert "datetime_utc" in rows[0] + + +class TestPayloadOrdering: + def test_tool_call_argument_order_is_preserved(self, store): + # canonical_json sorts keys - that is for hashing, not storage. + # Arguments come back in the order the model produced them. + messages = [ + llm.assistant( + ToolCallPart( + name="demo", + arguments={"timeout": 120, "options": ["tick"]}, + tool_call_id="tc1", + ) + ) + ] + tip = store.ensure_chain(messages) + assert list(store.load_chain(tip)[0].parts[0].arguments) == [ + "timeout", + "options", + ] + + def test_hashing_still_ignores_key_order(self, store): + one = store.ensure_chain( + [llm.assistant(ToolCallPart(name="d", arguments={"a": 1, "b": 2}))] + ) + two = store.ensure_chain( + [llm.assistant(ToolCallPart(name="d", arguments={"b": 2, "a": 1}))] + ) + assert one == two + assert store.db["messages"].count == 1 diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 57bd17991..781941370 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -460,7 +460,13 @@ def register_tools(self, register): ( log_row["prompt"], re.sub( - r"tc_[0-9a-z]{26}", "tc_TCID", json.dumps(log_row["tool_results"]) + r'"id": \d+', + '"id": ID', + re.sub( + r"tc_[0-9a-z]{26}", + "tc_TCID", + json.dumps(log_row["tool_results"]), + ), ), ) for log_row in log_rows @@ -469,12 +475,12 @@ def register_tools(self, register): ('{"tool_calls": [{"name": "upper", "arguments": {"text": "one"}}]}', "[]"), ( "", - '[{"id": 2, "tool_id": 1, "name": "upper", "output": "ONE", "tool_call_id": "tc_TCID", "exception": null, "attachments": []}]', + '[{"id": ID, "tool_id": 1, "name": "upper", "output": "ONE", "tool_call_id": "tc_TCID", "exception": null, "attachments": []}]', ), ('{"tool_calls": [{"name": "upper", "arguments": {"text": "two"}}]}', "[]"), ( "", - '[{"id": 3, "tool_id": 1, "name": "upper", "output": "TWO", "tool_call_id": "tc_TCID", "exception": null, "attachments": []}]', + '[{"id": ID, "tool_id": 1, "name": "upper", "output": "TWO", "tool_call_id": "tc_TCID", "exception": null, "attachments": []}]', ), ( '{"tool_calls": [{"name": "upper", "arguments": {"text": "three"}}]}', @@ -482,7 +488,7 @@ def register_tools(self, register): ), ( "", - '[{"id": 4, "tool_id": 1, "name": "upper", "output": "THREE", "tool_call_id": "tc_TCID", "exception": null, "attachments": []}]', + '[{"id": ID, "tool_id": 1, "name": "upper", "output": "THREE", "tool_call_id": "tc_TCID", "exception": null, "attachments": []}]', ), ) # Test the --td option From c692b93783093e8776c9c648ecaf797c82821c87 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 18:34:28 -0700 Subject: [PATCH 140/258] sqlite-utils>=3.39.1 So delete_where() works properly. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 36a6dc2f1..22c15252d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "condense-json>=0.1.3", "openai>=2.32.0", "click-default-group>=1.2.3", - "sqlite-utils>=3.37", + "sqlite-utils>=3.39.1", "sqlite-migrate==0.1a2", "pydantic>=2.0.0", "PyYAML", From 550535b3da634814c6634f202beb264863d33167 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 18:45:42 -0700 Subject: [PATCH 141/258] Green suite: fix the two bugs behind the 11 failing tests test_llm_default_prompt (9): the new read path returned options_json as null where the legacy responses table always recorded "{}". The turns table stores null for empty options - _dump({}) returns None - but the --json renderer only decodes non-null *_json keys, so the null leaked through to the output. _LogRowBuilder.build now normalises it to "{}" at read time; storage keeps null. test_tool_errors (2): the fifth ToolResultPart construction path the previous session could not find was _append_tool_results_to_chain, used by the chain-loop and resume paths, which dropped exception. It now formats it like the other four sites. Its attachment handling is left alone - the trailing user-role message there is deliberate. Also: a mypy annotation for log_row_extras' fragments dict, and black reformatting of four files that had drifted in the WIP commit. Co-Authored-By: Claude Fable 5 --- llm/logs.py | 18 ++++++++++-------- llm/models.py | 2 +- tests/test_llm_logs.py | 1 - tests/test_logs_store.py | 8 +++++++- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/llm/logs.py b/llm/logs.py index 9cbf955df..c86c32fdf 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -698,9 +698,7 @@ def build(self, row: dict) -> dict: outputs = self.store.load_chain(row["tip_message_hash"])[len(inputs) :] prompt_parts = inputs[-1].parts if inputs else [] - system_parts = ( - inputs[0].parts if inputs and inputs[0].role == "system" else [] - ) + system_parts = inputs[0].parts if inputs and inputs[0].role == "system" else [] out_parts = [part for message in outputs for part in message.parts] built = { @@ -723,6 +721,9 @@ def build(self, row: dict) -> dict: } built.update( { + # The turn stores null when no options were set; the + # responses table always recorded "{}". + "options_json": row["options_json"] or "{}", "prompt": _text_of(prompt_parts, TextPart), # None rather than "" when there was no system # message, matching what was recorded before. @@ -786,13 +787,11 @@ def log_rows( # rather than from the message text, so it matches what -f means. for index, fragment_hash in enumerate(fragment_hashes): key = f"fragment_{index}" - where.append( - f"""turns.id in ( + where.append(f"""turns.id in ( select turn_fragments.turn_id from turn_fragments join fragments on fragments.id = turn_fragments.fragment_id where fragments.hash = :{key} - )""" - ) + )""") params[key] = fragment_hash # A turn "used" a tool when a tool result was among its inputs, @@ -880,7 +879,10 @@ def log_row_extras(store: "LogStore", row: dict) -> dict: ) ] - fragments = {"prompt_fragments": [], "system_fragments": []} + fragments: dict[str, list[dict]] = { + "prompt_fragments": [], + "system_fragments": [], + } for fragment_row in store.db.query( """ select turn_fragments.kind, fragments.hash, fragments.content, diff --git a/llm/models.py b/llm/models.py index 6daa69557..89ed1f6f5 100644 --- a/llm/models.py +++ b/llm/models.py @@ -2728,6 +2728,7 @@ def _append_tool_results_to_chain(chain, tool_results, attachments) -> list[Any] name=tr.name, output=tr.output, tool_call_id=tr.tool_call_id, + exception=_format_tool_exception(tr.exception), ) for tr in tool_results ], @@ -3508,7 +3509,6 @@ def matches(self, query: str) -> bool: return any(query_lower in alias.lower() for alias in all_strings) - def _format_tool_exception(exception) -> str | None: """Render a tool's exception the way it is recorded. diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index 9fdd9f5fb..d18876ea4 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -154,7 +154,6 @@ def test_logs_text(log_path, usage): assert output == expected - @legacy_rows_only def test_logs_text_with_options(user_path): """Test that ## Options section appears when options_json is set""" diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 668f7d93b..3571ecaae 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -1049,7 +1049,13 @@ def test_filters_by_fragment(self, user_path, tmpdir): assert [r["prompt"] for r in rows] == ["FRAGMENT BODY\nwith fragment"] def test_filters_by_tool(self, user_path): - run("-m", "echo", '{"tool_calls": [{"name": "llm_version"}]}', "-T", "llm_version") + run( + "-m", + "echo", + '{"tool_calls": [{"name": "llm_version"}]}', + "-T", + "llm_version", + ) run("-m", "echo", "no tools here") forget_legacy(user_path) assert len(json.loads(run("logs", "-T", "llm_version", "--json").output)) == 1 From 16916ed39d396b137c6e67e658da77d63b05f8c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 19:05:28 -0700 Subject: [PATCH 142/258] Fold prompt=/fragments=/attachments=/tool_results= into explicit messages= Passing prompt text alongside messages= silently dropped it from prompt.messages: the model received text that never appeared in the chain, so the log disagreed with what the model saw. Verified against the echo model - the request carried "and a follow-up" while the logged chain held only the explicit history. messages= now means authoritative history, with the other prompt arguments folded in as the new turn's input - a tool-role message for tool_results, then a user message built from fragments + prompt text + attachments. This is the same convention reply() and the chain loop already used internally. The fold happens at the public entry points - Model.prompt, AsyncModel.prompt and _build_full_chain's explicit branch, which now shares the synthesized branch's _append_turn_input helper - not in the Prompt class, which still stores messages= verbatim. Internal callers pass pre-baked chains that already contain the prompt text, so folding in the property would double it. No plugin changes needed: llm-anthropic (main) and llm-gemini (llm-parts) read prompt.messages exclusively, so they simply start receiving the previously-missing user message; legacy plugins never look at prompt.messages at all. Two tests pinning the old "prompt= is ignored" behaviour are updated to assert the fold, and the behaviour change gets a changelog entry under a new Unreleased section. Co-Authored-By: Claude Fable 5 --- docs/changelog.md | 4 ++ llm/models.py | 121 +++++++++++++++++++++++++-------------- tests/test_logs_store.py | 14 +++++ tests/test_parts.py | 121 ++++++++++++++++++++++++++++++++++----- 4 files changed, 204 insertions(+), 56 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 47f617cd0..a204aba57 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- Passing `prompt=`, `fragments=`, `attachments=` or `tool_results=` alongside `messages=` to `model.prompt()` or `conversation.prompt()` now appends that new input to the supplied message history, instead of silently omitting it from `prompt.messages` - previously the model could receive text that never appeared in the logged conversation. [#1562](https://github.com/simonw/llm/pull/1562) + (v0_31_1)= ## 0.31.1 (2026-07-09) diff --git a/llm/models.py b/llm/models.py index 89ed1f6f5..d5a838fc6 100644 --- a/llm/models.py +++ b/llm/models.py @@ -545,6 +545,61 @@ def _wrap_tools(tools: list[ToolDef]) -> list[Tool]: return wrapped_tools +def _append_turn_input( + chain: list[Any], + prompt: str | None, + fragments=None, + attachments=None, + tool_results=None, +) -> list[Any]: + """Append a turn's new input to a message chain. + + A tool-role message for any tool results, then a user-role message + built from fragments + prompt text + attachments. This is what makes + ``messages=`` mean "authoritative history" without the other prompt + arguments being silently dropped from the chain. Mutates and returns + ``chain``. + """ + from .parts import ( + AttachmentPart, + Message, + TextPart, + ToolResultPart, + ) + + if tool_results: + chain.append( + Message( + role="tool", + parts=[ + ToolResultPart( + name=tr.name, + output=tr.output, + tool_call_id=tr.tool_call_id, + exception=_format_tool_exception(tr.exception), + attachments=list(tr.attachments or []), + ) + for tr in tool_results + ], + ) + ) + + user_parts: list[Any] = [] + # Fragments are concatenated into the prompt text before it is + # sent, so they have to be in the chain too - prompt.messages is + # meant to be exactly what the model sees. Matches Prompt.prompt. + prompt_text = "\n".join( + [str(fragment) for fragment in fragments or []] + ([prompt] if prompt else []) + ) + if prompt_text: + user_parts.append(TextPart(text=prompt_text)) + for att in attachments or []: + user_parts.append(AttachmentPart(attachment=att)) + if user_parts: + chain.append(Message(role="user", parts=user_parts)) + return chain + + def _combine_system(system, system_fragments): "Concatenate the system prompt and any system fragments into one string." bits = [ @@ -615,17 +670,19 @@ def _build_full_chain( exactly what the model sees. If ``explicit_messages`` is provided, the caller has opted out - of history reconstruction and the list is used as-is. + of history reconstruction: the list is the authoritative history, + and the new turn's input is appended to it. """ - from .parts import ( - AttachmentPart, - Message, - TextPart, - ToolResultPart, - ) + from .parts import Message, TextPart if explicit_messages is not None: - return list(explicit_messages) + return _append_turn_input( + list(explicit_messages), + prompt, + fragments, + attachments, + tool_results, + ) chain: list[Any] = [] if self.loaded_messages: @@ -648,40 +705,7 @@ def _build_full_chain( if system_text: chain.append(Message(role="system", parts=[TextPart(text=system_text)])) - # Append the new turn's input - if tool_results: - chain.append( - Message( - role="tool", - parts=[ - ToolResultPart( - name=tr.name, - output=tr.output, - tool_call_id=tr.tool_call_id, - exception=_format_tool_exception(tr.exception), - attachments=list(tr.attachments or []), - ) - for tr in tool_results - ], - ) - ) - - user_parts: list[Any] = [] - # Fragments are concatenated into the prompt text before it is - # sent, so they have to be in the chain too - prompt.messages is - # meant to be exactly what the model sees. Matches Prompt.prompt. - prompt_text = "\n".join( - [str(fragment) for fragment in fragments or []] - + ([prompt] if prompt else []) - ) - if prompt_text: - user_parts.append(TextPart(text=prompt_text)) - for att in attachments or []: - user_parts.append(AttachmentPart(attachment=att)) - if user_parts: - chain.append(Message(role="user", parts=user_parts)) - - return chain + return _append_turn_input(chain, prompt, fragments, attachments, tool_results) @dataclass @@ -3213,6 +3237,13 @@ def prompt( key_value = kwargs.pop("key", None) merged = _merge_options(options, kwargs) self._validate_attachments(attachments) + if messages is not None: + # messages= is the authoritative history; the other prompt + # arguments are this turn's new input, folded in so that + # response.prompt.messages stays exactly what the model sees. + messages = _append_turn_input( + list(messages), prompt, fragments, attachments, tool_results + ) return Response( Prompt( prompt, @@ -3332,6 +3363,12 @@ def prompt( key_value = kwargs.pop("key", None) merged = _merge_options(options, kwargs) self._validate_attachments(attachments) + if messages is not None: + # Same fold as Model.prompt: messages= is the history, the + # other prompt arguments are this turn's new input. + messages = _append_turn_input( + list(messages), prompt, fragments, attachments, tool_results + ) return AsyncResponse( Prompt( prompt, diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 3571ecaae..d421d5b59 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -728,6 +728,20 @@ def test_a_chain_writes_the_store_too(self, store, mock_model): assert store.db["turns"].count == 1 assert len(store.thread_messages(conversation.id)) == 2 + def test_messages_plus_prompt_both_reach_the_chain(self, store, mock_model): + """prompt= alongside messages= used to vanish from the logged + chain: Prompt.messages returned the explicit list verbatim, so + the text the model answered was absent from the store.""" + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("follow-up", messages=[llm.user("original")]) + response.text() + response.log_to_db(store.db) + turn = next(iter(store.db["turns"].rows)) + chain = store.load_chain(turn["tip_message_hash"]) + texts = [part.text for message in chain for part in message.parts] + assert texts == ["original", "follow-up", "Hello"] + assert store.verify() == [] + def test_successive_library_turns_extend_the_thread(self, store, mock_model): conversation = mock_model.conversation() for reply in ("One", "Two"): diff --git a/tests/test_parts.py b/tests/test_parts.py index 1f0771aa4..e96096b94 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -940,9 +940,11 @@ def test_explicit_messages_returned_verbatim(self, mock_model): assert p.messages == explicit def test_explicit_messages_ignores_prompt_kwarg(self, mock_model): - """Explicit messages= is authoritative. A prompt= string passed - alongside is no longer auto-appended — the invariant is that - prompt.messages equals exactly what the model was sent.""" + """Prompt stores messages= verbatim. Folding prompt= into the + chain happens in model.prompt() / _build_full_chain before the + Prompt is constructed — internal callers pass chains that + already contain the prompt text, so Prompt itself must not + append it a second time.""" from llm.models import Prompt explicit = [llm.system("x"), llm.user("prior"), llm.user("follow-up")] @@ -1001,6 +1003,90 @@ async def test_async_conversation_prompt_accepts_messages(self, async_mock_model assert response.prompt.messages == [llm.user("q")] +class TestModelPromptMessagesPlusNewInput: + """messages= is the authoritative history; prompt=, fragments=, + attachments= and tool_results= passed alongside are this turn's new + input, folded into the chain so prompt.messages still equals exactly + what the model sees. Previously the prompt text was silently absent + from the chain — and so from the log.""" + + def test_prompt_text_appends_user_message(self, mock_model): + mock_model.enqueue(["ok"]) + response = mock_model.prompt("follow-up", messages=[llm.user("original")]) + response.text() + assert response.prompt.messages == [ + llm.user("original"), + llm.user("follow-up"), + ] + + def test_fragments_join_the_prompt_text(self, mock_model): + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "question", fragments=["context"], messages=[llm.user("original")] + ) + response.text() + assert response.prompt.messages == [ + llm.user("original"), + llm.user("context\nquestion"), + ] + + def test_attachments_join_the_user_message(self, mock_model): + att = llm.Attachment(type="image/png", url="http://example.com/a.png") + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "look", attachments=[att], messages=[llm.user("original")] + ) + response.text() + assert response.prompt.messages == [ + llm.user("original"), + llm.Message( + role="user", + parts=[ + llm.parts.TextPart(text="look"), + llm.parts.AttachmentPart(attachment=att), + ], + ), + ] + + def test_tool_results_append_tool_message_before_user(self, mock_model): + tr = llm.ToolResult(name="t", output="ok", tool_call_id="c1") + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "next", messages=[llm.user("original")], tool_results=[tr] + ) + response.text() + assert response.prompt.messages == [ + llm.user("original"), + llm.Message( + role="tool", + parts=[ + llm.parts.ToolResultPart(name="t", output="ok", tool_call_id="c1") + ], + ), + llm.user("next"), + ] + + def test_conversation_prompt_folds_too(self, mock_model): + mock_model.enqueue(["ok"]) + conv = mock_model.conversation() + response = conv.prompt("follow-up", messages=[llm.user("original")]) + response.text() + assert response.prompt.messages == [ + llm.user("original"), + llm.user("follow-up"), + ] + + @pytest.mark.asyncio + async def test_async_model_prompt_folds_too(self, async_mock_model): + async_mock_model.enqueue(["ok"]) + response = async_mock_model.prompt("follow-up", messages=[llm.user("original")]) + await response.text() + assert response.prompt.messages == [ + llm.user("original"), + llm.user("follow-up"), + ] + + # Invariant: response.prompt.messages == exactly what the model was # sent for this turn, regardless of whether the caller used # model.prompt(messages=[...]), conversation.prompt("text"), or @@ -1008,17 +1094,21 @@ async def test_async_conversation_prompt_accepts_messages(self, async_mock_model class TestConversationFullChainInvariant: - def test_explicit_messages_is_authoritative_no_prompt_combine(self, mock_model): - """Explicit messages= is the whole list. If prompt= is ALSO - passed, it's ignored for messages-building — the caller asked - for exact control.""" + def test_explicit_messages_plus_prompt_appends_user_message(self, mock_model): + """Explicit messages= is the authoritative history. A prompt= + passed alongside is this turn's new input and is folded in as a + trailing user message — the model sees it, so the chain (and + therefore the log) must contain it.""" mock_model.enqueue(["ok"]) response = mock_model.prompt( - "this prompt argument is ignored", + "the new question", messages=[llm.user("q")], ) response.text() - assert response.prompt.messages == [llm.user("q")] + assert response.prompt.messages == [ + llm.user("q"), + llm.user("the new question"), + ] def test_conversation_second_turn_prompt_messages_has_full_chain(self, mock_model): mock_model.enqueue(["a1"]) @@ -1815,17 +1905,20 @@ def test_model_chain_accepts_messages(self, mock_model): r1 = chain._responses[0] assert r1.prompt.messages == [llm.user("explicit")] - def test_chain_messages_is_authoritative_over_prompt_kwarg(self, mock_model): - """Parity with prompt(): when both are passed, messages= wins - and the prompt= string is not folded into the chain.""" + def test_chain_messages_plus_prompt_folds_user_message(self, mock_model): + """Parity with prompt(): messages= is the history and a prompt= + passed alongside is appended as this turn's user message.""" mock_model.enqueue(["ok"]) chain = mock_model.chain( - "ignored text", + "the new question", messages=[llm.user("explicit")], ) chain.text() r1 = chain._responses[0] - assert r1.prompt.messages == [llm.user("explicit")] + assert r1.prompt.messages == [ + llm.user("explicit"), + llm.user("the new question"), + ] def test_chain_with_messages_and_prior_conversation(self, mock_model): """Explicit messages= on chain() replaces history reconstruction; From a9e505867d51d2f56115ba07a9102e2f74bf38ae Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 19:59:53 -0700 Subject: [PATCH 143/258] Merged read for llm logs, end of dual-write, message store docs llm logs now unions both generations of tables: legacy-only responses rows are shaped like the new-table rows and merged in by id, with dual-write-era twins (same id in responses and turns) suppressed in favour of the turn. Both id spaces are ULIDs, so a straight sort interleaves the histories chronologically and each side's top-N contains the union's top-N. All filters are implemented on both sides; -q/--query stays a clear error until search is designed for the new tables (a design document for that lives in plans/search-design.md). log_to_db no longer writes the legacy tables at all - it collapses to LogStore(db).log(response). The legacy tables remain as read-only history. Consequences carried through: - A response logged outside a conversation now gets a thread of its own, restoring the guarantee the conversations table provided - without it, library-logged responses could never be continued with llm -c. - load_conversation is thread-first: most recent conversation comes from a union of threads and conversations, and a conversation with no legacy row is reconstructed from its thread plus the model of its latest turn. - Tool reuse on llm -c reads tool names from turn_tools when there are no rebuilt legacy responses to copy prompt.tools from. - Raw provider payloads (prompt_json/response_json, including completion logprobs) are no longer persisted anywhere; the chain is the record. Toolbox instance provenance is likewise not recorded - tool execution provenance remains an open design decision. The 44 tests xfailed as "legacy rows only" pass again under the merged read and their marks are removed; the -q marks stay, applied per-param so the no-query case runs. Tests that asserted dual writes now assert the store alone, tests that used log_to_db to fabricate legacy rows seed them directly, and test_llm_default_prompt's fixture mystery from the WIP commit dissolves with the legacy block it lived in. Docs: new "The message store" section in docs/logging.md - schema prose, the hash contract, cog-generated worked examples driven through LogStore.log, a SQL cookbook adapting conversation-views.sql, and Response.log_to_db() documented as the supported write API with LogStore explicitly not yet stable. Co-Authored-By: Claude Fable 5 --- README.md | 9 + docs/changelog.md | 4 + docs/logging.md | 353 ++++++++++++++++++++++++++++++++ llm/cli.py | 104 ++++++++-- llm/logs.py | 296 +++++++++++++++++++++++++- llm/models.py | 223 ++------------------ tests/test_async_parity.py | 27 ++- tests/test_attachments.py | 17 +- tests/test_chat.py | 107 +++------- tests/test_chat_templates.py | 39 ++-- tests/test_cli_openai_models.py | 9 +- tests/test_llm.py | 124 +++++------ tests/test_llm_logs.py | 63 ++---- tests/test_logs_store.py | 77 ++++--- tests/test_parts.py | 25 ++- tests/test_plugins.py | 151 ++++---------- tests/test_tools.py | 44 ++-- 17 files changed, 1048 insertions(+), 624 deletions(-) diff --git a/README.md b/README.md index d3da2444e..d56a5d5cb 100644 --- a/README.md +++ b/README.md @@ -332,6 +332,15 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Browsing data collected using schemas](https://llm.datasette.io/en/stable/logging.html#browsing-data-collected-using-schemas) * [Browsing logs using Datasette](https://llm.datasette.io/en/stable/logging.html#browsing-logs-using-datasette) * [Backing up your database](https://llm.datasette.io/en/stable/logging.html#backing-up-your-database) + * [The message store](https://llm.datasette.io/en/stable/logging.html#the-message-store) + * [Threads, turns, messages and parts](https://llm.datasette.io/en/stable/logging.html#threads-turns-messages-and-parts) + * [A worked example](https://llm.datasette.io/en/stable/logging.html#a-worked-example) + * [Content addressing as a contract](https://llm.datasette.io/en/stable/logging.html#content-addressing-as-a-contract) + * [Forking and shared history](https://llm.datasette.io/en/stable/logging.html#forking-and-shared-history) + * [Storage by reference](https://llm.datasette.io/en/stable/logging.html#storage-by-reference) + * [Table by table](https://llm.datasette.io/en/stable/logging.html#table-by-table) + * [Querying the message store](https://llm.datasette.io/en/stable/logging.html#querying-the-message-store) + * [Logging from Python](https://llm.datasette.io/en/stable/logging.html#logging-from-python) * [SQL schema](https://llm.datasette.io/en/stable/logging.html#sql-schema) * [Related tools](https://llm.datasette.io/en/stable/related-tools.html) * [strip-tags](https://llm.datasette.io/en/stable/related-tools.html#strip-tags) diff --git a/docs/changelog.md b/docs/changelog.md index a204aba57..f663b15f2 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,10 @@ ## Unreleased +- `llm logs` now reads both generations of log tables: history recorded by older versions of llm in the legacy `responses` table is merged into the output alongside new turns, with rows from the dual-write era deduplicated by id. The `-m`, `-c`, `-f`, `-T`, `--tools` and `--schema` filters work across both. `-q/--query` full-text search is temporarily unavailable while search against the new tables is designed. [#1562](https://github.com/simonw/llm/pull/1562) +- Logging no longer writes the legacy tables: `Response.log_to_db()` records only the content-addressed message store. The legacy tables are kept in place as read-only history. Raw provider payloads (the old `prompt_json` and `response_json` columns) are no longer persisted - the stored message chain is the record of what was sent and returned. [#1562](https://github.com/simonw/llm/pull/1562) +- A response logged outside of a conversation now gets a thread of its own, so responses logged through the Python API can be continued with `llm -c` - the same guarantee the `conversations` table used to provide. [#1562](https://github.com/simonw/llm/pull/1562) +- New documentation for the message store: the schema, the content-addressing hash contract, worked examples and a SQL cookbook, in {ref}`the logging documentation `. [#1562](https://github.com/simonw/llm/pull/1562) - Passing `prompt=`, `fragments=`, `attachments=` or `tool_results=` alongside `messages=` to `model.prompt()` or `conversation.prompt()` now appends that new input to the supplied message history, instead of silently omitting it from `prompt.messages` - previously the model could receive text that never appeared in the logged conversation. [#1562](https://github.com/simonw/llm/pull/1562) (v0_31_1)= diff --git a/docs/logging.md b/docs/logging.md index 3fda94100..eb9a928f4 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -276,6 +276,359 @@ llm logs backup /tmp/backup.db ``` This uses SQLite [VACUUM INTO](https://sqlite.org/lang_vacuum.html#vacuum_with_an_into_clause) under the hood. +(logging-message-store)= + +## The message store + +The `logs.db` database contains two generations of tables. Databases created by older versions of LLM recorded everything in a `responses` table, with companion tables such as `prompt_attachments` and `tool_calls` hanging off it. Current versions write to a set of **content-addressed** tables instead: `threads`, `turns`, `messages` and `parts`. Content-addressed means that rows are identified by a hash of their content rather than an assigned id, so identical content is stored exactly once. + +The legacy tables are read-only history now. The `llm logs` command merges the two generations: rows that only exist in the legacy `responses` table are combined with rows from the new tables, so history recorded by an older version stays visible after an upgrade. All new logging writes only the content-addressed tables. + +(logging-message-store-vocabulary)= + +### Threads, turns, messages and parts + +From the top down: + +- A **thread** is a conversation. It is a named pointer at the message at the head of that conversation, and its id is the conversation id displayed by `llm logs`. +- A **turn** is a single model call within a thread. It records everything specific to that call - which model answered, options, token counts, timings - and points at the messages that were its input and output. +- A **message** is one entry in a conversation, with a role of `system`, `user` or `assistant`. Each message links to its parent, forming a chain, and is identified by a hash of its content combined with that parent link. +- A **part** is a piece of content within a message: text, reasoning, a tool call, a tool result or an attachment. A message's parts are ordered by their `position`. + +(logging-message-store-example)= + +### A worked example + +Here is a two turn conversation, logged to a fresh database and then dumped. It was generated with a small scripted model that returns canned replies - every model plugin logs through the same code path, so rows written by a real model have exactly the same shape, but the canned replies keep this example deterministic. + + +``` +messages and their parts: + +user b2:d6b0cd4e7a65ea90423c50fadb3f5704 + parent: null + part 0: {"type": "text", "text": "Suggest a name for a pet pelican"} + +assistant b2:0f2c02ad982050b623b7e034199c8c61 + parent: b2:d6b0cd4e7a65ea90423c50fadb3f5704 + part 0: {"type": "text", "text": "How about Percy? Pelicans suit a dignified name."} + +user b2:c785dd6c77540150c2647f406cacc76f + parent: b2:0f2c02ad982050b623b7e034199c8c61 + part 0: {"type": "text", "text": "Now one for a pet walrus"} + +assistant b2:a30a236e0d1b717c592e826d06e3c9d2 + parent: b2:c785dd6c77540150c2647f406cacc76f + part 0: {"type": "text", "text": "Wallace. It pairs nicely with Percy."} + +turns: + +turn $TURN_1_ID + thread_id: $THREAD_ID + parent_message_hash: b2:d6b0cd4e7a65ea90423c50fadb3f5704 + tip_message_hash: b2:0f2c02ad982050b623b7e034199c8c61 + model: scripted + +turn $TURN_2_ID + thread_id: $THREAD_ID + parent_message_hash: b2:c785dd6c77540150c2647f406cacc76f + tip_message_hash: b2:a30a236e0d1b717c592e826d06e3c9d2 + model: scripted + +thread: + +thread $THREAD_ID + name: Suggest a name for a pet pelican + tip_message_hash: b2:a30a236e0d1b717c592e826d06e3c9d2 +``` + + +Things to notice: + +- The four messages form a chain: the first has a `null` parent and each subsequent message names the hash of the one before it. +- The `part N:` lines show each part's `payload` column verbatim - the part serialized as JSON. +- Each turn brackets one model call. Its `parent_message_hash` is the tip of the chain that was sent to the model - the last input message, usually that turn's user prompt - and its `tip_message_hash` is the chain tip after the model's reply was appended. The prompt and response that `llm logs` displays are derived by splitting the chain at the parent, rather than being stored a second time. +- The thread's id is the conversation id, its name is derived from the first prompt and its `tip_message_hash` follows the head of the conversation as new turns are logged. +- The ULID identifiers - sortable random identifiers issued in time order, the same id space older versions used for response ids - and the timestamp columns have been replaced with placeholders because they change on every run. The hashes have not: they depend only on the message content and its position in the chain, so replaying this conversation produces these exact four hashes. + +(logging-message-store-hashes)= + +### Content addressing as a contract + +A message's hash is calculated like this: + +1. Build the object `{"parent": parent_hash, "message": message}`, where `parent_hash` is the hash of the previous message in the chain (or `null` for the first message) and `message` is the message's dictionary representation - its role, its parts and any provider metadata. +2. Serialize that object to canonical JSON: keys sorted, compact `,` and `:` separators with no extra whitespace, non-ASCII characters left unescaped. +3. Hash the UTF-8 encoding of that string with [BLAKE2b](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) using a 16 byte digest, and prefix the hex digest with `b2:`. + +The `b2:` prefix tags the hash with the algorithm that produced it. If a future version of LLM ever changes the digest or the canonical form, the change will be detectable instead of silently splitting the stored data into two incompatible halves. + +Two design decisions matter here: + +- **The hash covers resolved content.** {ref}`Fragment ` references and attachment references are expanded before hashing, so the hash always covers exactly the content the model saw, never the compressed form it happens to be stored in. The internal `LogStore.verify()` method re-derives every hash from the stored rows to check this stays true. +- **The parent hash participates in the hash.** The same content appearing at a different point in a conversation is a different node. This is what makes two conversations that share a prefix collapse to shared rows, with no explicit comparison required: replaying the same messages produces the same hashes, and a hash that is already present needs nothing written. + +A consequence of the second decision is that a stateless client - one that holds its own conversation history and re-sends the whole thing with every call - writes only the new tail on each request. It also makes forks cheap, as described next. Here the first conversation from the worked example is logged again, followed by a second conversation that starts with the same prompt and then diverges: + + +``` +message rows after the first conversation: 4 +message rows after both conversations: 6 + +rows added by the second conversation: + +user b2:371ceec468c0ec797ac7042970992c0b + parent: b2:0f2c02ad982050b623b7e034199c8c61 + +assistant b2:4ad19ddf94ea086aa9a8aa6fd8b0af05 + parent: b2:371ceec468c0ec797ac7042970992c0b +``` + + +Two conversations of two turns each - eight messages sent to the model in total - produced six rows. The second conversation's first turn hashed to rows that were already present, so only its second turn was written, and the new user message's parent is the assistant reply both conversations share. + +(logging-message-store-forking)= + +### Forking and shared history + +Because messages form a parent-linked tree, a conversation can fork: a new thread can point at any existing message and continue from there, sharing its entire history with the thread it came from until the two diverge. Nothing is copied when this happens. The `threads.forked_from` column records which thread a fork came from. + +TODO: description of diagram + +Shared rows cut both ways. Deleting a conversation is not the same as deleting its rows, because another thread may reach the same messages - removing them would silently corrupt that thread's history. For this reason LLM does not currently delete message rows at all; garbage collection of unreachable messages is deliberately left as future work. + +(logging-message-store-references)= + +### Storage by reference + +The hash of a message covers its resolved content, but storage is by reference. A text part whose content borrows from a fragment does not store a copy of that fragment. Instead of a `text` key its payload holds a `text_ref` list of fragment references and literal segments: + +```json +{"type": "text", "text_ref": [{"fragment": 1}, {"literal": "\nquestion about it"}]} +``` + +Here fragment `1` is an id in the existing `fragments` table. Reading the part concatenates the fragment content and the literal back together, reproducing the exact text that was hashed. Ask a hundred questions about a novel and the novel is stored once. + +Attachments work the same way: the binary content lives in the `attachments` table, keyed by a SHA-256 hash of the bytes, and the part payload stores that id in place of the data. + +(logging-message-store-tables)= + +### Table by table + +The full schema for these tables appears in {ref}`the SQL schema section ` below. + +- `messages` - one row per unique message. `hash` is the content address described above, `parent_hash` links to the previous message in the chain and `role` is `system`, `user` or `assistant`. `provider_metadata` holds any provider-specific data carried by the message; it participates in the hash. +- `parts` - the content of each message, ordered by `position`. `payload` is the part serialized as JSON - stored in the order the model produced it, not the canonical key-sorted form used for hashing. `type` and `tool_name` are copied out of the payload so they can be filtered on directly. +- `part_attachments` and `part_fragments` - junction tables recording which rows in `attachments` and `fragments` a part's payload references, in order. +- `threads` - one row per conversation. `id` is the conversation id, `tip_message_hash` points at the current head of the conversation and `forked_from` records the thread a fork came from. +- `turns` - one row per model call. `parent_message_hash` and `tip_message_hash` bracket the call's input and output as described above, and the remaining columns record provenance: model, options, schema, token counts and timings. Turn ids are ULIDs, in the same id space as legacy response ids, which is how the two generations of tables sort together in `llm logs`. +- `turn_tools` - which {ref}`tool ` definitions were available to a turn, referencing the `tools` table. +- `turn_fragments` - which fragments a turn was given, with their `kind` (`prompt` or `system`) and order. Provenance lives here rather than on the shared message rows, and this table is what powers `llm logs -f`. + +(logging-message-store-queries)= + +### Querying the message store + +These queries can be pasted into [Datasette](https://datasette.io/) or `sqlite3` against your `logs.db`. This one renders every conversation tree as indented text, resolving `text_ref` payloads back to their full text and stepping the indentation in at each fork: + +```sql +with recursive +siblings as ( + select + hash, + parent_hash, + count(*) over (partition by parent_hash) as branches + from messages +), +walk as ( + select + messages.hash, + 0 as indent, + printf('%08d', messages.rowid) as sort_key + from messages + where messages.parent_hash is null + union all + select + siblings.hash, + walk.indent + (siblings.branches > 1), + walk.sort_key || '/' || printf('%08d', messages.rowid) + from siblings + join messages on messages.hash = siblings.hash + join walk on siblings.parent_hash = walk.hash +) +select + substr(' ', 1, walk.indent * 4) + || messages.role || ': ' + || coalesce( + json_extract(parts.payload, '$.text'), + ( + select group_concat( + coalesce( + json_extract(piece.value, '$.literal'), + (select content from fragments + where id = json_extract(piece.value, '$.fragment')) + ), + '' order by piece.key + ) + from json_each(json_extract(parts.payload, '$.text_ref')) as piece + ), + parts.type + ) as entry +from walk +join messages on messages.hash = walk.hash +left join parts on parts.message_hash = messages.hash +order by walk.sort_key, parts.position; +``` + +Every turn that was given a specific fragment, via the `turn_fragments` provenance table - set `:fragment_hash` to a hash from `llm fragments`: + +```sql +select turns.id, turns.model, turns.datetime_utc, turn_fragments.kind +from turns +join turn_fragments on turn_fragments.turn_id = turns.id +join fragments on fragments.id = turn_fragments.fragment_id +where fragments.hash = :fragment_hash +order by turns.id; +``` + +The most recently active conversations, with a count of their turns: + +```sql +select + threads.id, + threads.name, + count(turns.id) as num_turns, + max(turns.datetime_utc) as last_used +from threads +left join turns on turns.thread_id = threads.id +group by threads.id +order by last_used desc +limit 10; +``` + +(logging-message-store-python)= + +### Logging from Python + +The supported way to write to a log database from Python is the `log_to_db()` method on a response, which is also what plugins should call: + +```python +import llm +import sqlite_utils + +db = sqlite_utils.Database("logs.db") +model = llm.get_model("gpt-4o-mini") +response = model.prompt("A short pelican fact") +print(response.text()) +response.log_to_db(db) +``` + +`log_to_db()` takes a `sqlite_utils.Database` and records the response's thread, turn, messages, parts, fragments, attachments and tools in the content-addressed tables. It applies any outstanding migrations itself, so it is safe to call against a brand new database file or one created by an older version of LLM. The underlying `LogStore` class is internal and its API may change - `log_to_db()` and the table schema documented on this page are the supported interfaces. + (logging-sql-schema)= ## SQL schema diff --git a/llm/cli.py b/llm/cli.py index 0b7a8e18a..9c49ccf96 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -62,7 +62,12 @@ ) from llm.models import ChainResponse, _BaseConversation -from .logs import LogStore, log_row_extras, log_rows +from .logs import ( + LogStore, + legacy_log_row_extras, + log_row_extras, + merged_log_rows, +) from .migrations import migrate from .plugins import load_plugins, pm from .utils import ( @@ -1320,8 +1325,16 @@ def load_conversation( db = sqlite_utils.Database(log_path) migrate(db) if conversation_id is None: - # Return the most recent conversation, or None if there are none - matches = list(db["conversations"].rows_where(order_by="id desc", limit=1)) + # Most recent conversation from either generation of tables - + # thread ids are conversation ids, so the union dedupes rows + # from the dual-write era. + matches = list(db.query(""" + select id from ( + select id from threads + union + select id from conversations + ) order by id desc limit 1 + """)) if matches: conversation_id = matches[0]["id"] else: @@ -1329,7 +1342,30 @@ def load_conversation( try: row = cast(sqlite_utils.db.Table, db["conversations"]).get(conversation_id) except sqlite_utils.db.NotFoundError: - raise click.ClickException(f"No conversation found with id={conversation_id}") + # No legacy record - reconstruct the equivalent from the thread + # and its most recent turn's model. + try: + thread_row = cast(sqlite_utils.db.Table, db["threads"]).get(conversation_id) + except sqlite_utils.db.NotFoundError: + raise click.ClickException( + f"No conversation found with id={conversation_id}" + ) + model_match = next( + db.query( + "select model from turns where thread_id = ? order by id desc limit 1", + [conversation_id], + ), + None, + ) + if model_match is None: + raise click.ClickException( + f"No conversation found with id={conversation_id}" + ) + row = { + "id": conversation_id, + "name": thread_row["name"], + "model": model_match["model"], + } # Inflate that conversation conversation_class = AsyncConversation if async_ else Conversation response_class = AsyncResponse if async_ else Response @@ -1361,6 +1397,23 @@ def load_conversation( except KeyError: pass + # Plugin tools recorded against the first turn, for the same + # reuse-on-continue behaviour the rebuilt responses provide. + conversation.loaded_tools = [ + tool_row["name"] + for tool_row in db.query( + """ + select tools.name from tools + join turn_tools on turn_tools.tool_id = tools.id + where tools.plugin is not null + and turn_tools.turn_id = ( + select id from turns where thread_id = ? order by id limit 1 + ) + """, + [conversation_id], + ) + ] + return conversation @@ -1720,11 +1773,15 @@ def logs_list( if current_conversation: try: - conversation_id = next( - db.query( - "select conversation_id from responses order by id desc limit 1" - ) - )["conversation_id"] + # Thread ids are conversation ids and both id spaces are + # ULIDs, so the most recent of either world wins. + conversation_id = next(db.query(""" + select conversation_id from ( + select thread_id as conversation_id, id from turns + union all + select conversation_id, id from responses + ) order by id desc limit 1 + """))["conversation_id"] except StopIteration: # No conversations yet raise click.ClickException("No conversations found") @@ -1754,8 +1811,9 @@ def logs_list( schema_id = make_schema_id(schema)[0] if schema else None - rows = log_rows( - LogStore(db), + store = LogStore(db) + rows = merged_log_rows( + store, count=count if count and count > 0 else None, model_id=model_id, thread_id=conversation_id, @@ -1771,9 +1829,19 @@ def logs_list( if not data: rows.reverse() - # Attachments, fragments and tool info, all from the new tables. - store = LogStore(db) - extras_by_id = {row["id"]: log_row_extras(store, row) for row in rows} + # Attachments, fragments and tool info. New rows carry theirs in + # the row's parts; legacy rows batch-fetch from the legacy tables. + legacy_extras = legacy_log_row_extras( + db, [row["id"] for row in rows if row.get("_legacy")] + ) + extras_by_id = { + row["id"]: ( + legacy_extras[row["id"]] + if row.get("_legacy") + else log_row_extras(store, row) + ) + for row in rows + } attachments_by_id = { id: extras["attachments"] for id, extras in extras_by_id.items() } @@ -4023,9 +4091,15 @@ def _gather_tools( def _get_conversation_tools(conversation, tools): - if conversation and not tools and conversation.responses: + if not conversation or tools: + return None + if conversation.responses: # Copy plugin tools from first response in conversation initial_tools = conversation.responses[0].prompt.tools if initial_tools: # Only tools from plugins: return [tool.name for tool in initial_tools if tool.plugin] + elif conversation.loaded_tools: + # Conversation loaded from the message store - the tool names + # were read from turn_tools instead of rebuilt responses. + return list(conversation.loaded_tools) diff --git a/llm/logs.py b/llm/logs.py index c86c32fdf..2c577bada 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -380,13 +380,16 @@ def log(self, response, thread_id: str | None = None) -> str: """ if thread_id is None: conversation = getattr(response, "conversation", None) - if conversation is not None: - thread_id = self.ensure_thread( - conversation.id, - name=_conversation_name( - response.prompt.prompt or response.prompt.system or "" - ), - ) + # A response logged outside any conversation still gets a + # thread of its own, so `llm -c` and `llm logs` can always + # find it - the same guarantee the conversations table used + # to provide. + thread_id = self.ensure_thread( + conversation.id if conversation else str(monotonic_ulid()).lower(), + name=_conversation_name( + response.prompt.prompt or response.prompt.system or "" + ), + ) prompt_fragments = list(response.prompt.fragments or []) system_fragments = list(response.prompt.system_fragments or []) @@ -761,8 +764,8 @@ def log_rows( ) -> list[dict]: """Rows for `llm logs`, newest first, drawn from the new tables. - Deliberately blind to anything logged before this schema existed - - those conversations have no turns, so they simply do not appear. + Sees only conversations with turns - merged_log_rows adds the rows + that exist solely in the legacy `responses` table. """ where: list[str] = [] params: dict[str, Any] = {} @@ -939,3 +942,278 @@ def _tool_ids_by_name(store: "LogStore") -> dict: tool_row["name"]: tool_row["id"] for tool_row in store.db.query("select id, name from tools") } + + +# -- legacy rows --------------------------------------------------------- +# +# History logged by older versions of llm lives only in the `responses` +# table. Those rows are merged into `llm logs` output, shaped like the +# rows _LogRowBuilder produces. A response whose id also exists in +# `turns` is suppressed - that is what a dual-write-era row or a +# backfilled conversation looks like, and the turn is the richer record. + +LEGACY_LOG_ROWS_SQL = """ +select + responses.id, + responses.model, + responses.resolved_model, + responses.prompt, + responses.system, + responses.prompt_json, + responses.options_json, + responses.response, + responses.reasoning, + responses.response_json, + responses.conversation_id, + responses.duration_ms, + responses.datetime_utc, + responses.input_tokens, + responses.output_tokens, + responses.token_details, + conversations.name as conversation_name, + conversations.model as conversation_model, + schemas.content as schema_json +from responses +left join schemas on responses.schema_id = schemas.id +left join conversations on responses.conversation_id = conversations.id +where responses.id not in (select id from turns){extra_where} +order by responses.id desc{limit} +""" + + +def legacy_log_rows( + db, + *, + count: int | None = None, + model_id: str | None = None, + thread_id: str | None = None, + fragment_hashes=(), + tool_names=(), + any_tools: bool = False, + schema_id: str | None = None, + id_gt: str | None = None, + id_gte: str | None = None, +) -> list[dict]: + """Rows for `llm logs` that exist only in the legacy tables. + + Applies the same filters as log_rows, translated to the legacy + schema. Thread ids are conversation ids, so the two filters match + the same conversations on either side of the upgrade. + """ + where: list[str] = [] + params: dict[str, Any] = {} + + if model_id: + where.append("(responses.model = :model or responses.resolved_model = :model)") + params["model"] = model_id + if thread_id: + where.append("responses.conversation_id = :thread_id") + params["thread_id"] = thread_id + if id_gt: + where.append("responses.id > :id_gt") + params["id_gt"] = id_gt + if id_gte: + where.append("responses.id >= :id_gte") + params["id_gte"] = id_gte + if schema_id: + where.append("responses.schema_id = :schema_id") + params["schema_id"] = schema_id + + for index, fragment_hash in enumerate(fragment_hashes): + key = f"fragment_{index}" + where.append(f"""exists ( + select 1 from prompt_fragments + where prompt_fragments.response_id = responses.id + and prompt_fragments.fragment_id in ( + select fragments.id from fragments where hash = :{key} + ) + union + select 1 from system_fragments + where system_fragments.response_id = responses.id + and system_fragments.fragment_id in ( + select fragments.id from fragments where hash = :{key} + ) + )""") + params[key] = fragment_hash + + if any_tools: + where.append("""exists ( + select 1 from tool_results + where tool_results.response_id = responses.id + )""") + for index, tool_name in enumerate(tool_names): + key = f"tool_{index}" + where.append(f"""exists ( + select 1 from tool_results + join tools on tools.id = tool_results.tool_id + where tool_results.response_id = responses.id + and tools.name = :{key} + )""") + params[key] = tool_name + + sql = LEGACY_LOG_ROWS_SQL.format( + extra_where=(" and " + " and ".join(where)) if where else "", + limit=f" limit {count}" if count else "", + ) + rows = [dict(row) for row in db.query(sql, params)] + for row in rows: + row["_legacy"] = True + return rows + + +def merged_log_rows(store: "LogStore", *, count: int | None = None, **filters): + """Rows for `llm logs`: the new tables plus legacy-only responses. + + Both sides are newest-first and ids are ULIDs on both sides of the + upgrade, so a straight sort interleaves the two histories + chronologically and the top ``count`` of the union is always within + the top ``count`` of each side. + """ + rows = log_rows(store, count=count, **filters) + rows.extend(legacy_log_rows(store.db, count=count, **filters)) + rows.sort(key=lambda row: row["id"], reverse=True) + if count: + rows = rows[:count] + return rows + + +LEGACY_ATTACHMENTS_SQL = """ +select + response_id, + attachments.id, + attachments.type, + attachments.path, + attachments.url, + length(attachments.content) as content_length +from attachments +join prompt_attachments + on attachments.id = prompt_attachments.attachment_id +where prompt_attachments.response_id in ({placeholders}) +order by prompt_attachments."order" +""" + +LEGACY_FRAGMENTS_SQL = """ +select + {table}.response_id, + fragments.hash, + fragments.id as fragment_id, + fragments.content, + ( + select json_group_array(fragment_aliases.alias) + from fragment_aliases + where fragment_aliases.fragment_id = fragments.id + ) as aliases +from {table} +join fragments on {table}.fragment_id = fragments.id +where {table}.response_id in ({placeholders}) +order by {table}."order" +""" + +LEGACY_TOOLS_SQL = """ +select responses.id, + coalesce( + (select json_group_array(json_object( + 'id', t.id, + 'hash', t.hash, + 'name', t.name, + 'description', t.description, + 'input_schema', json(t.input_schema) + )) + from tools t + join tool_responses tr on t.id = tr.tool_id + where tr.response_id = responses.id + ), + '[]' + ) as tools, + coalesce( + (select json_group_array(json_object( + 'id', tc.id, + 'tool_id', tc.tool_id, + 'name', tc.name, + 'arguments', json(tc.arguments), + 'tool_call_id', tc.tool_call_id + )) + from tool_calls tc + where tc.response_id = responses.id + ), + '[]' + ) as tool_calls, + coalesce( + (select json_group_array(json_object( + 'id', tr.id, + 'tool_id', tr.tool_id, + 'name', tr.name, + 'output', tr.output, + 'tool_call_id', tr.tool_call_id, + 'exception', tr.exception, + 'attachments', coalesce( + (select json_group_array(json_object( + 'id', a.id, + 'type', a.type, + 'path', a.path, + 'url', a.url, + 'content', a.content + )) + from tool_results_attachments tra + join attachments a on tra.attachment_id = a.id + where tra.tool_result_id = tr.id + ), + '[]' + ) + )) + from tool_results tr + where tr.response_id = responses.id + ), + '[]' + ) as tool_results +from responses +where id in ({placeholders}) +""" + + +def legacy_log_row_extras(db, ids: list[str]) -> dict[str, dict]: + """Extras for legacy rows, batch-fetched, keyed by response id. + + Same shape as log_row_extras, with each entry shaped the way the + pre-turns `llm logs` rendered it. + """ + extras: dict[str, dict] = { + id: { + "attachments": [], + "prompt_fragments": [], + "system_fragments": [], + "tools": [], + "tool_calls": [], + "tool_results": [], + } + for id in ids + } + if not ids: + return extras + placeholders = ",".join("?" * len(ids)) + + for attachment in db.query( + LEGACY_ATTACHMENTS_SQL.format(placeholders=placeholders), ids + ): + attachment = dict(attachment) + response_id = attachment.pop("response_id") + extras[response_id]["attachments"].append(attachment) + + for table in ("prompt_fragments", "system_fragments"): + for fragment in db.query( + LEGACY_FRAGMENTS_SQL.format(table=table, placeholders=placeholders), ids + ): + fragment = dict(fragment) + response_id = fragment.pop("response_id") + extras[response_id][table].append(fragment) + + for row in db.query(LEGACY_TOOLS_SQL.format(placeholders=placeholders), ids): + extras[row["id"]].update( + { + "tools": json.loads(row["tools"]), + "tool_calls": json.loads(row["tool_calls"]), + "tool_results": json.loads(row["tool_results"]), + } + ) + + return extras diff --git a/llm/models.py b/llm/models.py index d5a838fc6..e9c6ab27d 100644 --- a/llm/models.py +++ b/llm/models.py @@ -28,7 +28,6 @@ ) import httpx -from condense_json import condense_json from .errors import NeedsKeyException from .serialization import ResponseDict @@ -43,9 +42,6 @@ from .utils import ( Fragment, - ensure_fragment, - ensure_tool, - make_schema_id, mimetype_from_path, mimetype_from_string, monotonic_ulid, @@ -636,6 +632,11 @@ class _BaseConversation: # exact message list, so reasoning signatures and provider metadata # survive being reloaded. loaded_messages: list[Any] | None = None + # Names of plugin tools recorded against this conversation's first + # turn in storage. Read when the conversation was loaded from the + # message store, where there are no rebuilt responses to copy + # prompt.tools from. + loaded_tools: list[str] | None = None @classmethod @abstractmethod @@ -1480,215 +1481,15 @@ def token_usage(self) -> str: ) def log_to_db(self, db): - # Built up front because it applies migrations, which have to run - # before the inserts below create any tables implicitly. + # Everything - thread, turn, messages, parts, fragments, + # attachments, tools - is recorded in the content-addressed + # tables. The legacy tables are no longer written; they hold + # history logged by older versions and are still read. + # This lives here rather than in the CLI because log_to_db() + # is what plugins call. from .logs import LogStore - store = LogStore(db) - - conversation = self.conversation - if not conversation: - conversation = Conversation(model=self.model) - db["conversations"].insert( - { - "id": conversation.id, - "name": _conversation_name( - self.prompt.prompt or self.prompt.system or "" - ), - "model": conversation.model.model_id, - }, - ignore=True, - ) - schema_id = None - if self.prompt.schema: - schema_id, schema_json = make_schema_id(self.prompt.schema) - db["schemas"].insert({"id": schema_id, "content": schema_json}, ignore=True) - - response_id = self.id - replacements = {} - # Include replacements from previous responses - for previous_response in conversation.responses[:-1]: - for fragment in (previous_response.prompt.fragments or []) + ( - previous_response.prompt.system_fragments or [] - ): - fragment_id = ensure_fragment(db, fragment) - replacements[f"f:{fragment_id}"] = fragment - replacements[f"r:{previous_response.id}"] = ( - previous_response.text_or_raise() - ) - - for i, fragment in enumerate(self.prompt.fragments): - fragment_id = ensure_fragment(db, fragment) - replacements[f"f{fragment_id}"] = fragment - db["prompt_fragments"].insert( - { - "response_id": response_id, - "fragment_id": fragment_id, - "order": i, - }, - ) - for i, fragment in enumerate(self.prompt.system_fragments): - fragment_id = ensure_fragment(db, fragment) - replacements[f"f{fragment_id}"] = fragment - db["system_fragments"].insert( - { - "response_id": response_id, - "fragment_id": fragment_id, - "order": i, - }, - ) - - response_text = self.text_or_raise() - replacements[f"r:{response_id}"] = response_text - # Concatenate visible reasoning text from the assembled - # ReasoningPart entries; redacted markers contribute nothing. - from .parts import ReasoningPart - - reasoning_text = "".join( - p.text - for m in self._messages_now() - for p in m.parts - if isinstance(p, ReasoningPart) and p.text - ) - json_data = self.json() - - response = { - "id": response_id, - "model": self.model.model_id, - "prompt": self.prompt._prompt, - "system": self.prompt._system, - "prompt_json": condense_json(self._prompt_json, replacements), - "options_json": { - key: value - for key, value in dict(self.prompt.options).items() - if value is not None - }, - "response": response_text, - "reasoning": reasoning_text or None, - "response_json": condense_json(json_data, replacements), - "conversation_id": conversation.id, - "duration_ms": self.duration_ms(), - "datetime_utc": self.datetime_utc(), - "input_tokens": self.input_tokens, - "output_tokens": self.output_tokens, - "token_details": ( - json.dumps(self.token_details) if self.token_details else None - ), - "schema_id": schema_id, - "resolved_model": self.resolved_model, - } - db["responses"].insert(response) - - # Persist any attachments - loop through with index - for index, attachment in enumerate(self.prompt.attachments): - attachment_id = attachment.id() - db["attachments"].insert( - { - "id": attachment_id, - "type": attachment.resolve_type(), - "path": attachment.path, - "url": attachment.url, - "content": attachment.content, - }, - replace=True, - ) - db["prompt_attachments"].insert( - { - "response_id": response_id, - "attachment_id": attachment_id, - "order": index, - }, - ) - - # Persist any tools, tool calls and tool results - tool_ids_by_name = {} - for tool in self.prompt.tools: - tool_id = ensure_tool(db, tool) - tool_ids_by_name[tool.name] = tool_id - db["tool_responses"].insert( - { - "tool_id": tool_id, - "response_id": response_id, - } - ) - for tool_call in self.tool_calls(): # TODO Should be _or_raise() - db["tool_calls"].insert( - { - "response_id": response_id, - "tool_id": tool_ids_by_name.get(tool_call.name) or None, - "name": tool_call.name, - "arguments": json.dumps(tool_call.arguments), - "tool_call_id": tool_call.tool_call_id, - } - ) - for tool_result in self.prompt.tool_results: - instance_id = None - if tool_result.instance: - try: - if not tool_result.instance.instance_id: - tool_result.instance.instance_id = ( - db["tool_instances"] - .insert( - { - "plugin": tool.plugin, - "name": tool.name.split("_")[0], - "arguments": json.dumps( - tool_result.instance._config - ), - } - ) - .last_pk - ) - instance_id = tool_result.instance.instance_id - except AttributeError: - pass - tool_result_id = ( - db["tool_results"] - .insert( - { - "response_id": response_id, - "tool_id": tool_ids_by_name.get(tool_result.name) or None, - "name": tool_result.name, - "output": tool_result.output, - "tool_call_id": tool_result.tool_call_id, - "instance_id": instance_id, - "exception": ( - ( - f"{tool_result.exception.__class__.__name__}: {tool_result.exception!s}" - ) - if tool_result.exception - else None - ), - } - ) - .last_pk - ) - # Persist attachments for tool results - for index, attachment in enumerate(tool_result.attachments): - attachment_id = attachment.id() - db["attachments"].insert( - { - "id": attachment_id, - "type": attachment.resolve_type(), - "path": attachment.path, - "url": attachment.url, - "content": attachment.content, - }, - replace=True, - ) - db["tool_results_attachments"].insert( - { - "tool_result_id": tool_result_id, - "attachment_id": attachment_id, - "order": index, - }, - ) - - # Mirror into the content-addressed tables. This lives here - # rather than in the CLI because log_to_db() is what plugins - # call - anything that logs a response should populate both - # representations, not just `llm` itself. - store.log(self) + LogStore(db).log(self) def _response_to_dict(response: "_BaseResponse") -> ResponseDict: diff --git a/tests/test_async_parity.py b/tests/test_async_parity.py index a84e567a6..7ba073fe6 100644 --- a/tests/test_async_parity.py +++ b/tests/test_async_parity.py @@ -95,15 +95,28 @@ async def test_async_from_row_response_messages_synthesized(tmp_path): from llm.migrations import migrate - model = llm.get_async_model("echo") - r = model.prompt("hello") - await r.text() - db = sqlite_utils.Database(str(tmp_path / "logs.db")) migrate(db) - # to_sync_response is what log_to_db uses for async. - sync_r = await r.to_sync_response() - sync_r.log_to_db(db) + # log_to_db no longer writes the legacy tables, so seed a row the + # way an older version of llm would have recorded it - from_row is + # the reader for exactly that history. + db["responses"].insert( + { + "id": "01aaaaaaaaaaaaaaaaaaaaaaaa", + "model": "echo", + "prompt": "hello", + "system": None, + "prompt_json": None, + "options_json": "{}", + "response": "echoed text", + "response_json": None, + "conversation_id": None, + "duration_ms": 1, + "datetime_utc": "2025-01-01T00:00:00", + "schema_id": None, + }, + alter=True, + ) row = next(db["responses"].rows) rehydrated = llm.AsyncResponse.from_row(db, row) diff --git a/tests/test_attachments.py b/tests/test_attachments.py index 88f9af3a6..d523e4745 100644 --- a/tests/test_attachments.py +++ b/tests/test_attachments.py @@ -44,12 +44,11 @@ def test_prompt_attachment(mock_model, logs_db, attachment_type, attachment_cont ) # Check it was logged correctly - conversations = list(logs_db["conversations"].rows) - assert len(conversations) == 1 - conversation = conversations[0] - assert conversation["model"] == "mock" - assert conversation["name"] == "describe file" - response = next(iter(logs_db["responses"].rows)) + threads = list(logs_db["threads"].rows) + assert len(threads) == 1 + assert threads[0]["name"] == "describe file" + turn = next(iter(logs_db["turns"].rows)) + assert turn["model"] == "mock" attachment = next(iter(logs_db["attachments"].rows)) assert attachment == { "id": ANY, @@ -58,9 +57,9 @@ def test_prompt_attachment(mock_model, logs_db, attachment_type, attachment_cont "url": None, "content": attachment_content, } - prompt_attachment = next(iter(logs_db["prompt_attachments"].rows)) - assert prompt_attachment["attachment_id"] == attachment["id"] - assert prompt_attachment["response_id"] == response["id"] + # The attachment hangs off a part of the stored user message + part_attachment = next(iter(logs_db["part_attachments"].rows)) + assert part_attachment["attachment_id"] == attachment["id"] def _count_open_fds(): diff --git a/tests/test_chat.py b/tests/test_chat.py index 098161330..0fd5b06fc 100644 --- a/tests/test_chat.py +++ b/tests/test_chat.py @@ -9,6 +9,27 @@ from click.testing import CliRunner import llm.cli +from llm.logs import LogStore, merged_log_rows + + +def logged_rows(db): + """Chronological log rows from the store, reduced to the fields + these tests care about.""" + rows = merged_log_rows(LogStore(db)) + rows.reverse() + return [ + { + "model": row["model"], + "prompt": row["prompt"], + "system": row["system"], + "options_json": row["options_json"], + "response": row["response"], + "conversation_id": row["conversation_id"], + "input_tokens": row["input_tokens"], + "output_tokens": row["output_tokens"], + } + for row in rows + ] @pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows") @@ -37,52 +58,30 @@ def test_chat_basic(mock_model, logs_db): "\n" ) # Should have logged - conversations = list(logs_db["conversations"].rows) - assert conversations[0] == { - "id": ANY, - "name": "Hi", - "model": "mock", - } - conversation_id = conversations[0]["id"] - responses = list(logs_db["responses"].rows) + threads = list(logs_db["threads"].rows) + assert threads[0]["name"] == "Hi" + conversation_id = threads[0]["id"] + responses = logged_rows(logs_db) assert responses == [ { - "id": ANY, "model": "mock", - "resolved_model": None, "prompt": "Hi", "system": None, - "prompt_json": None, "options_json": "{}", "response": "one world", - "response_json": None, "conversation_id": conversation_id, - "duration_ms": ANY, - "datetime_utc": ANY, "input_tokens": 1, "output_tokens": 1, - "token_details": None, - "schema_id": None, - "reasoning": None, }, { - "id": ANY, "model": "mock", - "resolved_model": None, "prompt": "Hi two", "system": None, - "prompt_json": None, "options_json": "{}", "response": "one again", - "response_json": None, "conversation_id": conversation_id, - "duration_ms": ANY, - "datetime_utc": ANY, "input_tokens": 2, "output_tokens": 1, - "token_details": None, - "schema_id": None, - "reasoning": None, }, ] # Now continue that conversation @@ -105,33 +104,17 @@ def test_chat_basic(mock_model, logs_db): "\n> quit" "\n" ) - new_responses = list( - logs_db.query( - "select * from responses where id not in ({})".format( - ", ".join("?" for _ in responses) - ), - [r["id"] for r in responses], - ) - ) + new_responses = logged_rows(logs_db)[len(responses) :] assert new_responses == [ { - "id": ANY, "model": "mock", - "resolved_model": None, "prompt": "Continue", "system": None, - "prompt_json": None, "options_json": "{}", "response": "continued", - "response_json": None, "conversation_id": conversation_id, - "duration_ms": ANY, - "datetime_utc": ANY, "input_tokens": 1, "output_tokens": 1, - "token_details": None, - "schema_id": None, - "reasoning": None, } ] @@ -157,26 +140,17 @@ def test_chat_system(mock_model, logs_db): "\n> quit" "\n" ) - responses = list(logs_db["responses"].rows) + responses = logged_rows(logs_db) assert responses == [ { - "id": ANY, "model": "mock", - "resolved_model": None, "prompt": "Hi", "system": "You are mean", - "prompt_json": None, "options_json": "{}", "response": "I am mean", - "response_json": None, "conversation_id": ANY, - "duration_ms": ANY, - "datetime_utc": ANY, "input_tokens": 1, "output_tokens": 1, - "token_details": None, - "schema_id": None, - "reasoning": None, } ] @@ -201,45 +175,27 @@ def test_chat_options(mock_model, logs_db, user_path): input="Hi with override\nquit\n", ) assert result.exit_code == 0 - responses = list(logs_db["responses"].rows) + responses = logged_rows(logs_db) assert responses == [ { - "id": ANY, "model": "mock", - "resolved_model": None, "prompt": "Hi", "system": None, - "prompt_json": None, "options_json": '{"max_tokens": 5}', "response": "Default options response", - "response_json": None, "conversation_id": ANY, - "duration_ms": ANY, - "datetime_utc": ANY, "input_tokens": 1, "output_tokens": 1, - "token_details": None, - "schema_id": None, - "reasoning": None, }, { - "id": ANY, "model": "mock", - "resolved_model": None, "prompt": "Hi with override", "system": None, - "prompt_json": None, "options_json": '{"max_tokens": 10}', "response": "Override options response", - "response_json": None, "conversation_id": ANY, - "duration_ms": ANY, - "datetime_utc": ANY, "input_tokens": 3, "output_tokens": 1, - "token_details": None, - "schema_id": None, - "reasoning": None, }, ] @@ -284,7 +240,10 @@ def test_chat_multi(mock_model, logs_db, input, expected): llm.cli.cli, ["chat", "-m", "mock", "--option", "max_tokens", "10"], input=input ) assert result.exit_code == 0 - rows = list(logs_db["responses"].rows_where(select="prompt, response")) + rows = [ + {"prompt": row["prompt"], "response": row["response"]} + for row in logged_rows(logs_db) + ] assert rows == expected @@ -311,7 +270,7 @@ def test_llm_chat_creates_log_database(tmpdir, monkeypatch, custom_database_path else: assert (user_path / "logs.db").exists() db_path = str(user_path / "logs.db") - assert sqlite_utils.Database(db_path)["responses"].count == 2 + assert sqlite_utils.Database(db_path)["turns"].count == 2 @pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows") diff --git a/tests/test_chat_templates.py b/tests/test_chat_templates.py index 56404ff44..ec8c49122 100644 --- a/tests/test_chat_templates.py +++ b/tests/test_chat_templates.py @@ -4,6 +4,7 @@ from click.testing import CliRunner import llm.cli +from llm.logs import LogStore, merged_log_rows @pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows") @@ -26,7 +27,7 @@ def test_chat_template_system_only_no_duplicate_prompt( assert result.exit_code == 0 # Ensure the logged prompt is not duplicated (no "hi\nhi") - rows = list(logs_db["responses"].rows) + rows = merged_log_rows(LogStore(logs_db)) assert len(rows) == 1 assert rows[0]["prompt"] == "hi" assert rows[0]["system"] == "Speak in French" @@ -51,17 +52,17 @@ def test_chat_system_fragments_only_first_turn(tmpdir, mock_model, logs_db): ) assert result.exit_code == 0 - # Verify only the first response has the system fragment - responses = list(logs_db["responses"].rows) - assert len(responses) == 2 - first_id = responses[0]["id"] - second_id = responses[1]["id"] + # Verify only the first turn has the system fragment + turns = list(logs_db["turns"].rows_where(order_by="id")) + assert len(turns) == 2 + first_id = turns[0]["id"] + second_id = turns[1]["id"] - sys_frags = list(logs_db["system_fragments"].rows) - # Exactly one system fragment row, attached to the first response only + sys_frags = list(logs_db["turn_fragments"].rows_where("kind = 'system'")) + # Exactly one system fragment row, attached to the first turn only assert len(sys_frags) == 1 - assert sys_frags[0]["response_id"] == first_id - assert sys_frags[0]["response_id"] != second_id + assert sys_frags[0]["turn_id"] == first_id + assert sys_frags[0]["turn_id"] != second_id @pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows") @@ -81,22 +82,22 @@ def test_chat_template_loads_tools_into_logs(logs_db, templates_path): ) assert result.exit_code == 0 - # Verify a single response was logged for the conversation - responses = list(logs_db["responses"].rows) - assert len(responses) == 1 - assert responses[0]["prompt"] == "hi" - response_id = responses[0]["id"] + # Verify a single turn was logged for the conversation + log_rows = merged_log_rows(LogStore(logs_db)) + assert len(log_rows) == 1 + assert log_rows[0]["prompt"] == "hi" + turn_id = log_rows[0]["id"] - # Tools from the template should be recorded against that response + # Tools from the template should be recorded against that turn rows = list( logs_db.query( """ select tools.name from tools - join tool_responses tr on tr.tool_id = tools.id - where tr.response_id = ? + join turn_tools tt on tt.tool_id = tools.id + where tt.turn_id = ? order by tools.name """, - [response_id], + [turn_id], ) ) assert [r["name"] for r in rows] == ["llm_time", "llm_version"] diff --git a/tests/test_cli_openai_models.py b/tests/test_cli_openai_models.py index f42cef977..ac37c0db0 100644 --- a/tests/test_cli_openai_models.py +++ b/tests/test_cli_openai_models.py @@ -6,6 +6,7 @@ import llm from llm.cli import cli +from llm.logs import LogStore @pytest.fixture @@ -442,6 +443,8 @@ def test_gpt4o_mini_sync_and_async(monkeypatch, tmpdir, httpx_mock, async_, usag # Confirm it was correctly logged assert log_db.exists() db = sqlite_utils.Database(str(log_db)) - assert db["responses"].count == 1 - row = next(db["responses"].rows) - assert row["response"] == "Ho ho ho" + assert db["turns"].count == 1 + turn = next(db["turns"].rows) + store = LogStore(db) + chain = store.load_chain(turn["tip_message_hash"]) + assert chain[-1].parts[0].text == "Ho ho ho" diff --git a/tests/test_llm.py b/tests/test_llm.py index f1d1afc68..4cee8d294 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -42,7 +42,7 @@ def test_llm_prompt_creates_log_database( else: assert (user_path / "logs.db").exists() db_path = str(user_path / "logs.db") - assert sqlite_utils.Database(db_path)["responses"].count == 1 + assert sqlite_utils.Database(db_path)["turns"].count == 1 @mock.patch.dict(os.environ, {"OPENAI_API_KEY": "X"}) @@ -64,7 +64,9 @@ def test_llm_default_prompt( # Reset the log_path database log_path = user_path / "logs.db" log_db = sqlite_utils.Database(str(log_path)) - log_db["responses"].delete_where() + if "turns" in log_db.table_names(): + with log_db.conn: + log_db.execute("delete from turns") logs_off_path = user_path / "logs-off" if logs_off: @@ -96,32 +98,20 @@ def test_llm_default_prompt( last_request = mocked_openai_chat.get_requests()[-1] assert last_request.headers["Authorization"] == "Bearer X" - # Was it logged? - rows = list(log_db["responses"].rows) + # Was it logged? The legacy tables are read-only now, so the turn + # is the record; its content is asserted below through `llm logs`. + rows = list(log_db["turns"].rows) if not should_log: assert len(rows) == 0 + assert log_db["responses"].count == 0 return assert len(rows) == 1 - expected = { - "model": "gpt-4o-mini", - "prompt": "three names \nfor a pet pelican", - "system": None, - "options_json": "{}", - "response": "Bob, Alice, Eve", - } row = rows[0] - assert expected.items() <= row.items() + assert row["model"] == "gpt-4o-mini" assert isinstance(row["duration_ms"], int) assert isinstance(row["datetime_utc"], str) - assert json.loads(row["prompt_json"]) == { - "messages": [{"role": "user", "content": "three names \nfor a pet pelican"}] - } - assert json.loads(row["response_json"]) == { - "choices": [{"message": {"content": {"$": f"r:{row['id']}"}}}], - "model": "gpt-4o-mini", - } # Test "llm logs" log_result = runner.invoke( @@ -174,7 +164,9 @@ def test_llm_prompt_continue(httpx_mock, user_path, async_): log_path = user_path / "logs.db" log_db = sqlite_utils.Database(str(log_path)) - log_db["responses"].delete_where() + if "turns" in log_db.table_names(): + with log_db.conn: + log_db.execute("delete from turns") # First prompt runner = CliRunner() @@ -186,7 +178,7 @@ def test_llm_prompt_continue(httpx_mock, user_path, async_): assert result.output == "Bob, Alice, Eve\n" # Should be logged - rows = list(log_db["responses"].rows) + rows = list(log_db["turns"].rows) assert len(rows) == 1 # Now ask a follow-up @@ -195,7 +187,7 @@ def test_llm_prompt_continue(httpx_mock, user_path, async_): assert result2.exit_code == 0, result2.output assert result2.output == "Terry\n" - rows = list(log_db["responses"].rows) + rows = list(log_db["turns"].rows) assert len(rows) == 2 @@ -236,7 +228,9 @@ def test_openai_chat_stream(mocked_openai_chat_stream, user_path): def test_openai_completion(mocked_openai_completion, user_path): log_path = user_path / "logs.db" log_db = sqlite_utils.Database(str(log_path)) - log_db["responses"].delete_where() + if "turns" in log_db.table_names(): + with log_db.conn: + log_db.execute("delete from turns") runner = CliRunner() result = runner.invoke( cli, @@ -263,18 +257,22 @@ def test_openai_completion(mocked_openai_completion, user_path): } # Check it was logged - rows = list(log_db["responses"].rows) + rows = list(log_db["turns"].rows) assert len(rows) == 1 - expected = { - "model": "gpt-3.5-turbo-instruct", - "prompt": "Say this is a test", - "system": None, - "prompt_json": '{"messages": ["Say this is a test"]}', - "options_json": "{}", - "response": "\n\nThis is indeed a test", - } - row = rows[0] - assert expected.items() <= row.items() + assert rows[0]["model"] == "gpt-3.5-turbo-instruct" + log_result = runner.invoke( + cli, ["logs", "-n", "1", "--json"], catch_exceptions=False + ) + log_json = json.loads(log_result.output) + assert ( + log_json[0].items() + >= { + "model": "gpt-3.5-turbo-instruct", + "prompt": "Say this is a test", + "system": None, + "response": "\n\nThis is indeed a test", + }.items() + ) def test_openai_completion_system_prompt_error(): @@ -303,7 +301,9 @@ def test_openai_completion_logprobs_stream( ): log_path = user_path / "logs.db" log_db = sqlite_utils.Database(str(log_path)) - log_db["responses"].delete_where() + if "turns" in log_db.table_names(): + with log_db.conn: + log_db.execute("delete from turns") runner = CliRunner() args = [ "-m", @@ -318,22 +318,11 @@ def test_openai_completion_logprobs_stream( result = runner.invoke(cli, args, catch_exceptions=False) assert result.exit_code == 0 assert result.output == "\n\nHi.\n" - rows = list(log_db["responses"].rows) + # Raw provider payloads (which carried the logprobs) are no longer + # persisted - the message chain is the record of what happened. + rows = list(log_db["turns"].rows) assert len(rows) == 1 - row = rows[0] - assert json.loads(row["response_json"]) == { - "content": {"$": f'r:{row["id"]}'}, - "logprobs": [ - {"text": "\n\n", "top_logprobs": [{"\n\n": -0.6, "\n": -1.9}]}, - {"text": "Hi", "top_logprobs": [{"Hi": -1.1, "Hello": -0.7}]}, - {"text": ".", "top_logprobs": [{".": -1.1, "!": -0.9}]}, - {"text": "", "top_logprobs": []}, - ], - "id": "cmpl-80MdSaou7NnPuff5ZyRMysWBmgSPS", - "object": "text_completion", - "model": "gpt-3.5-turbo-instruct", - "created": 1695097702, - } + assert rows[0]["model"] == "gpt-3.5-turbo-instruct" def test_openai_completion_logprobs_nostream( @@ -341,7 +330,9 @@ def test_openai_completion_logprobs_nostream( ): log_path = user_path / "logs.db" log_db = sqlite_utils.Database(str(log_path)) - log_db["responses"].delete_where() + if "turns" in log_db.table_names(): + with log_db.conn: + log_db.execute("delete from turns") runner = CliRunner() args = [ "-m", @@ -357,33 +348,12 @@ def test_openai_completion_logprobs_nostream( result = runner.invoke(cli, args, catch_exceptions=False) assert result.exit_code == 0 assert result.output == "\n\nHi.\n" - rows = list(log_db["responses"].rows) + # Raw provider payloads (which carried the logprobs) are no longer + # persisted - the message chain is the record of what happened. + rows = list(log_db["turns"].rows) assert len(rows) == 1 row = rows[0] - assert json.loads(row["response_json"]) == { - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "logprobs": { - "text_offset": [16, 18, 20], - "token_logprobs": [-0.6, -1.1, -0.9], - "tokens": ["\n\n", "Hi", "1"], - "top_logprobs": [ - {"\n": -1.9, "\n\n": -0.6}, - {"Hello": -0.7, "Hi": -1.1}, - {"!": -1.1, ".": -0.9}, - ], - }, - "text": {"$": f"r:{row['id']}"}, - } - ], - "created": 1695097747, - "id": "cmpl-80MeBfKJutM0uMNJkRrebJLeP3bxL", - "model": "gpt-3.5-turbo-instruct", - "object": "text_completion", - "usage": {"completion_tokens": 3, "prompt_tokens": 5, "total_tokens": 8}, - } + assert row["model"] == "gpt-3.5-turbo-instruct" EXTRA_MODELS_YAML = """ @@ -882,7 +852,7 @@ def test_llm_prompt_continue_with_database( else: assert (user_path / "logs.db").exists() db_path = str(user_path / "logs.db") - assert sqlite_utils.Database(db_path)["responses"].count == 2 + assert sqlite_utils.Database(db_path)["turns"].count == 2 def test_default_exports(): diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index d18876ea4..9b939aaca 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -17,15 +17,6 @@ from llm.migrations import migrate from llm.utils import monotonic_ulid -# These tests write rows straight into the legacy `responses` table and -# then assert on `llm logs` output. `llm logs` now reads the -# content-addressed tables only, so rows that exist nowhere else are -# invisible to it. Kept rather than deleted because whether the legacy -# read path comes back is still undecided. -legacy_rows_only = pytest.mark.xfail( - reason="llm logs no longer reads the legacy responses table", -) - # -q/--query was backed by responses_fts, which is legacy-only. Full # text search against the new tables has not been designed yet. search_not_supported = pytest.mark.xfail( @@ -106,7 +97,6 @@ def schema_log_path(user_path): id_re = re.compile(r"id: \w+") -@legacy_rows_only @pytest.mark.parametrize("usage", (False, True)) def test_logs_text(log_path, usage): runner = CliRunner() @@ -154,7 +144,6 @@ def test_logs_text(log_path, usage): assert output == expected -@legacy_rows_only def test_logs_text_with_options(user_path): """Test that ## Options section appears when options_json is set""" log_path = str(user_path / "logs_with_options.db") @@ -191,7 +180,6 @@ def test_logs_text_with_options(user_path): assert "- media_resolution: low" in output -@legacy_rows_only def test_logs_token_usage_details_are_markdown_code(user_path): log_path = str(user_path / "logs_token_details.db") db = sqlite_utils.Database(log_path) @@ -227,7 +215,6 @@ def test_logs_token_usage_details_are_markdown_code(user_path): ) in result.output -@legacy_rows_only @pytest.mark.parametrize("n", (None, 0, 2)) def test_logs_json(n, log_path): "Test that logs command correctly returns requested -n records" @@ -247,7 +234,6 @@ def test_logs_json(n, log_path): assert len(logs) == expected_length -@legacy_rows_only @pytest.mark.parametrize( "args", (["-r"], ["--response"], ["list", "-r"], ["list", "--response"]) ) @@ -259,7 +245,6 @@ def test_logs_response_only(args, log_path): assert result.output == 'response\n```python\nprint("hello word")\n```\n' -@legacy_rows_only @pytest.mark.parametrize( "args", ( @@ -281,7 +266,6 @@ def test_logs_extract_first_code(args, log_path): assert result.output == 'print("hello word")\n\n' -@legacy_rows_only @pytest.mark.parametrize( "args", ( @@ -301,7 +285,6 @@ def test_logs_extract_last_code(args, log_path): assert result.output == 'print("hello word")\n\n' -@legacy_rows_only @pytest.mark.parametrize("arg", ("-s", "--short")) @pytest.mark.parametrize("usage", (None, "-u", "--usage")) def test_logs_short(log_path, arg, usage): @@ -385,21 +368,24 @@ def test_logs_filtered(user_path, model, path_option): assert all(record["model"] == model for record in records) -@search_not_supported @pytest.mark.parametrize( "query,extra_args,expected", ( # With no search term order should be by datetime ("", [], ["doc1", "doc2", "doc3"]), # With a search it's order by rank instead - ("llama", [], ["doc1", "doc3"]), - ("alpaca", [], ["doc2"]), + pytest.param("llama", [], ["doc1", "doc3"], marks=search_not_supported), + pytest.param("alpaca", [], ["doc2"], marks=search_not_supported), # Model filter should work too - ("llama", ["-m", "davinci"], ["doc1", "doc3"]), - ("llama", ["-m", "davinci2"], []), + pytest.param( + "llama", ["-m", "davinci"], ["doc1", "doc3"], marks=search_not_supported + ), + pytest.param("llama", ["-m", "davinci2"], [], marks=search_not_supported), # Adding -l/--latest should return latest first (order by id desc) - ("llama", ["-l"], ["doc3", "doc1"]), - ("llama", ["--latest"], ["doc3", "doc1"]), + pytest.param("llama", ["-l"], ["doc3", "doc1"], marks=search_not_supported), + pytest.param( + "llama", ["--latest"], ["doc3", "doc1"], marks=search_not_supported + ), ), ) def test_logs_search(user_path, query, extra_args, expected): @@ -428,7 +414,6 @@ def _insert(id, text): assert [record["id"] for record in records] == expected -@legacy_rows_only @pytest.mark.parametrize( "args,expected", ( @@ -477,8 +462,6 @@ def test_logs_schema(schema_log_path, args, expected): assert result.output == expected -@legacy_rows_only -@legacy_rows_only def test_logs_schema_data_ids(schema_log_path): db = sqlite_utils.Database(schema_log_path) ulid = ULID.from_timestamp(time.time() + 100) @@ -669,7 +652,6 @@ def make_response(name, prompt_fragment_ids=None, system_fragment_ids=None): } -@legacy_rows_only @pytest.mark.parametrize( "fragment_refs,expected", ( @@ -775,7 +757,6 @@ def test_logs_fragments(fragments_fixture, fragment_refs, expected): assert reshaped2 == expected -@legacy_rows_only def test_logs_fragments_markdown(fragments_fixture): fragments_log_path = fragments_fixture["path"] fragment_hashes_by_slug = fragments_fixture["fragment_hashes_by_slug"] @@ -958,7 +939,6 @@ def test_logs_fragments_markdown(fragments_fixture): assert output.strip() == expected_output.strip() -@legacy_rows_only @pytest.mark.parametrize("arg", ("-e", "--expand")) def test_expand_fragment_json(fragments_fixture, arg): fragments_log_path = fragments_fixture["path"] @@ -980,7 +960,6 @@ def test_expand_fragment_json(fragments_fixture, arg): assert len(fragment2) > 200 -@legacy_rows_only def test_expand_fragment_markdown(fragments_fixture): fragments_log_path = fragments_fixture["path"] fragment_hashes_by_slug = fragments_fixture["fragment_hashes_by_slug"] @@ -1130,10 +1109,10 @@ def test_logs_resolved_model(logs_db, mock_model, async_mock_model, async_): ) assert result.exit_code == 0 # Should have logged the resolved model name - assert logs_db["responses"].count - response = next(iter(logs_db["responses"].rows)) - assert response["model"] == "mock" - assert response["resolved_model"] == "resolved-mock" + assert logs_db["turns"].count + turn = next(iter(logs_db["turns"].rows)) + assert turn["model"] == "mock" + assert turn["resolved_model"] == "resolved-mock" # Should show up in the JSON logs result2 = runner.invoke(cli, ["logs", "--json"]) @@ -1153,8 +1132,9 @@ def test_logs_resolved_model(logs_db, mock_model, async_mock_model, async_): def test_log_to_db_persists_visible_reasoning(logs_db, mock_model): """A response that streams reasoning events should round-trip the - visible reasoning text via the new responses.reasoning column.""" + visible reasoning text via a ReasoningPart in the stored chain.""" import llm + from llm.logs import LogStore, merged_log_rows mock_model.enqueue( [ @@ -1167,22 +1147,23 @@ def test_log_to_db_persists_visible_reasoning(logs_db, mock_model): response.text() response.log_to_db(logs_db) - row = next(logs_db["responses"].rows) + row = merged_log_rows(LogStore(logs_db))[0] assert row["response"] == "hello" assert row["reasoning"] == "thinking hard" def test_log_to_db_persists_empty_reasoning_when_absent(logs_db, mock_model): - """No reasoning emitted → empty/null reasoning column, never raises.""" + """No reasoning emitted → null reasoning, never raises.""" + from llm.logs import LogStore, merged_log_rows + mock_model.enqueue(["just text"]) response = mock_model.prompt("hi") response.text() response.log_to_db(logs_db) - row = next(logs_db["responses"].rows) - assert not row.get("reasoning") + row = merged_log_rows(LogStore(logs_db))[0] + assert not row["reasoning"] -@legacy_rows_only def test_logs_markdown_renders_reasoning_heading(user_path): """When a row has reasoning text, `llm logs` renders a `## Reasoning` heading between System and Response.""" diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index d421d5b59..1bbbe9ae7 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -15,6 +15,7 @@ import llm from llm.cli import cli from llm.logs import LogStore, canonical_json, message_hash +from llm.migrations import migrate from llm.models import Attachment from llm.parts import ( AttachmentPart, @@ -544,14 +545,17 @@ def test_successive_turns_extend_the_same_thread(self, store, mock_model): message.parts[0].text for message in store.thread_messages(conversation.id) ] == ["Hi", "Hello", "Hi", "Hello again"] - def test_a_response_without_a_conversation_creates_no_thread( + def test_a_response_without_a_conversation_gets_its_own_thread( self, store, mock_model ): + # Parity with the legacy tables, which recorded a conversation + # for every response - without this, a response logged through + # the library API could never be continued with `llm -c`. mock_model.enqueue(["Hello"]) response = mock_model.prompt("Hi") response.text() store.log(response) - assert store.db["threads"].count == 0 + assert store.db["threads"].count == 1 def test_an_explicit_thread_id_wins(self, store, mock_model): thread_id = store.create_thread(name="Mine") @@ -579,14 +583,15 @@ def run(*args): return result -class TestCliDualWrite: +class TestCliWrites: def test_a_prompt_writes_a_turn(self, cli_store): run("-m", "echo", "Hi") assert cli_store.db["turns"].count == 1 - def test_a_prompt_still_writes_the_legacy_tables(self, cli_store): + def test_a_prompt_does_not_write_the_legacy_tables(self, cli_store): run("-m", "echo", "Hi") - assert cli_store.db["responses"].count == 1 + assert cli_store.db["responses"].count == 0 + assert cli_store.db["conversations"].count == 0 def test_the_turn_points_at_the_stored_chain(self, cli_store): run("-m", "echo", "Hi") @@ -594,9 +599,9 @@ def test_the_turn_points_at_the_stored_chain(self, cli_store): chain = cli_store.load_chain(turn["tip_message_hash"]) assert [message.role for message in chain] == ["user", "assistant"] - def test_the_thread_matches_the_conversation(self, cli_store): + def test_the_thread_has_a_tip(self, cli_store): run("-m", "echo", "Hi") - conversation_id = next(iter(cli_store.db["conversations"].rows))["id"] + conversation_id = next(iter(cli_store.db["threads"].rows))["id"] assert cli_store.thread_tip(conversation_id) is not None def test_no_log_writes_nothing(self, cli_store): @@ -610,19 +615,14 @@ def test_continuing_extends_the_same_thread(self, cli_store): run("-m", "echo", "First") run("-m", "echo", "Second", "-c") assert cli_store.db["threads"].count == 1 - conversation_id = next(iter(cli_store.db["conversations"].rows))["id"] + conversation_id = next(iter(cli_store.db["threads"].rows))["id"] assert len(cli_store.thread_messages(conversation_id)) == 4 def test_history_comes_from_the_new_tables(self, user_path): - # Delete the legacy rows the old continuation path reads from. If - # `-c` still sends the full history, it can only have come from - # the content-addressed tables. run("-m", "echo", "First") db = sqlite_utils.Database(str(user_path / "logs.db")) - conversation_id = next(iter(db["conversations"].rows))["id"] - with db.conn: - db.execute("delete from responses") + conversation_id = next(iter(db["threads"].rows))["id"] db.close() run("-m", "echo", "Second", "-c") @@ -681,15 +681,31 @@ class TestLegacyConversations: def test_continuing_a_conversation_with_no_thread_still_works(self, user_path): # Conversations logged before this schema existed have no thread, # so `-c` has to fall back to rebuilding from the legacy rows. - run("-m", "echo", "First") - + # Nothing writes those rows any more - seed them the way an + # older version of llm would have. path = str(user_path / "logs.db") db = sqlite_utils.Database(path) - with db.conn: - db.execute("delete from threads") - db.execute("delete from turns") - db.execute("delete from parts") - db.execute("delete from messages") + migrate(db) + db["conversations"].insert( + {"id": "01aaaaaaaaaaaaaaaaaaaaaaaa", "name": "First", "model": "echo"} + ) + db["responses"].insert( + { + "id": "01aaaaaaaaaaaaaaaaaaaaaaab", + "model": "echo", + "prompt": "First", + "system": None, + "prompt_json": None, + "options_json": "{}", + "response": "First response", + "response_json": None, + "conversation_id": "01aaaaaaaaaaaaaaaaaaaaaaaa", + "duration_ms": 1, + "datetime_utc": "2025-01-01T00:00:00", + "schema_id": None, + }, + alter=True, + ) db.close() result = run("-m", "echo", "Second", "-c") @@ -712,12 +728,27 @@ def test_log_to_db_writes_the_store_too(self, store, mock_model): assert store.db["turns"].count == 1 assert store.db["messages"].count == 2 - def test_log_to_db_still_writes_the_legacy_tables(self, store, mock_model): + def test_log_to_db_leaves_the_legacy_tables_alone(self, store, mock_model): + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + response.log_to_db(store.db) + assert store.db["responses"].count == 0 + assert store.db["conversations"].count == 0 + + def test_log_to_db_without_conversation_still_gets_a_thread( + self, store, mock_model + ): + # The legacy path recorded a conversation for every response; + # the store keeps that guarantee with a thread, so `llm -c` can + # continue a response logged through the library API. mock_model.enqueue(["Hello"]) response = mock_model.prompt("Hi") response.text() response.log_to_db(store.db) - assert store.db["responses"].count == 1 + assert store.db["threads"].count == 1 + turn = next(iter(store.db["turns"].rows)) + assert turn["thread_id"] is not None def test_a_chain_writes_the_store_too(self, store, mock_model): conversation = mock_model.conversation() diff --git a/tests/test_parts.py b/tests/test_parts.py index e96096b94..d3da32876 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -1215,13 +1215,28 @@ def test_from_row_response_messages_synthesized_from_chunks( from llm.migrations import migrate - mock_model.enqueue(["answer text"]) - r1 = mock_model.prompt("q1") - r1.text() - db = sqlite_utils.Database(str(tmp_path / "logs.db")) migrate(db) - r1.log_to_db(db) + # log_to_db no longer writes the legacy tables - seed the row + # the way an older version of llm recorded it, since from_row + # is the reader for exactly that history. + db["responses"].insert( + { + "id": "01aaaaaaaaaaaaaaaaaaaaaaaa", + "model": "mock", + "prompt": "q1", + "system": None, + "prompt_json": None, + "options_json": "{}", + "response": "answer text", + "response_json": None, + "conversation_id": None, + "duration_ms": 1, + "datetime_utc": "2025-01-01T00:00:00", + "schema_id": None, + }, + alter=True, + ) # Rehydrate the response row = next(db["responses"].rows) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 781941370..f82791dcf 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -801,11 +801,7 @@ def after_call(tool, tool_call, tool_result): ) # Test the logging worked - rows = list(logs_db.query(TOOL_RESULTS_SQL)) - # JSON decode things in rows - for row in rows: - row["tool_calls"] = json.loads(row["tool_calls"]) - row["tool_results"] = json.loads(row["tool_results"]) + rows = tool_activity_rows(logs_db) assert rows == [ { "model": "echo", @@ -822,24 +818,8 @@ def after_call(tool, tool_call, tool_result): "model": "echo", "tool_calls": [], "tool_results": [ - { - "name": "Memory_set", - "output": "null", - "instance": { - "name": "Memory", - "plugin": "ToolboxPlugin", - "arguments": "{}", - }, - }, - { - "name": "Memory_get", - "output": "two", - "instance": { - "name": "Memory", - "plugin": "ToolboxPlugin", - "arguments": "{}", - }, - }, + {"name": "Memory_set", "output": "null"}, + {"name": "Memory_get", "output": "two"}, ], }, { @@ -854,11 +834,6 @@ def after_call(tool, tool_call, tool_result): { "name": "Filesystem_list_files", "output": json.dumps([str(other_path)]), - "instance": { - "name": "Filesystem", - "plugin": "ToolboxPlugin", - "arguments": json.dumps({"path": str(my_dir2)}), - }, } ], }, @@ -935,11 +910,7 @@ def test_toolbox_logging_async(logs_db, tmpdir): plugins.pm.unregister(name="ToolboxPlugin") # Check the database - rows = list(logs_db.query(TOOL_RESULTS_SQL)) - # JSON decode things in rows - for row in rows: - row["tool_calls"] = json.loads(row["tool_calls"]) - row["tool_results"] = json.loads(row["tool_results"]) + rows = tool_activity_rows(logs_db) assert rows == [ { "model": "echo", @@ -954,33 +925,9 @@ def test_toolbox_logging_async(logs_db, tmpdir): "model": "echo", "tool_calls": [], "tool_results": [ - { - "name": "Memory_set", - "output": "null", - "instance": { - "name": "Filesystem", - "plugin": "ToolboxPlugin", - "arguments": "{}", - }, - }, - { - "name": "Memory_get", - "output": "two", - "instance": { - "name": "Filesystem", - "plugin": "ToolboxPlugin", - "arguments": "{}", - }, - }, - { - "name": "Filesystem_list_files", - "output": "[]", - "instance": { - "name": "Filesystem", - "plugin": "ToolboxPlugin", - "arguments": json.dumps({"path": str(path)}), - }, - }, + {"name": "Memory_set", "output": "null"}, + {"name": "Memory_get", "output": "two"}, + {"name": "Filesystem_list_files", "output": "[]"}, ], }, ] @@ -1011,57 +958,33 @@ def test_plugins_command(): ] -TOOL_RESULTS_SQL = """ --- First, create ordered subqueries for tool_calls and tool_results -with ordered_tool_calls as ( - select - tc.response_id, - json_group_array( - json_object( - 'name', tc.name, - 'arguments', tc.arguments - ) - ) as tool_calls_json - from ( - select * from tool_calls order by id - ) tc - where tc.id is not null - group by tc.response_id -), -ordered_tool_results as ( - select - tr.response_id, - json_group_array( - json_object( - 'name', tr.name, - 'output', tr.output, - 'instance', case - when ti.id is not null then json_object( - 'name', ti.name, - 'plugin', ti.plugin, - 'arguments', ti.arguments - ) - else null - end - ) - ) as tool_results_json - from ( - select distinct tr.*, ti.id as ti_id, ti.name as ti_name, - ti.plugin, ti.arguments as ti_arguments - from tool_results tr - left join tool_instances ti on tr.instance_id = ti.id - order by tr.id - ) tr - left join tool_instances ti on tr.instance_id = ti.id - where tr.id is not null - group by tr.response_id -) -select - r.model, - coalesce(otc.tool_calls_json, '[]') as tool_calls, - coalesce(otr.tool_results_json, '[]') as tool_results -from responses r -left join ordered_tool_calls otc on r.id = otc.response_id -left join ordered_tool_results otr on r.id = otr.response_id -group by r.id, r.model -order by r.id""" +def tool_activity_rows(db): + """Per-turn tool calls and results from the message store, in the + shape the old TOOL_RESULTS_SQL produced from the legacy tables. + + Toolbox instance provenance is absent: the store does not record + which instance served a call - tool execution provenance is still + an open design decision. + """ + from llm.logs import LogStore, log_row_extras, merged_log_rows + + store = LogStore(db) + rows = merged_log_rows(store) + rows.reverse() + out = [] + for row in rows: + extras = log_row_extras(store, row) + out.append( + { + "model": row["model"], + "tool_calls": [ + {"name": call["name"], "arguments": json.dumps(call["arguments"])} + for call in extras["tool_calls"] + ], + "tool_results": [ + {"name": result["name"], "output": result["output"]} + for result in extras["tool_results"] + ], + } + ) + return out diff --git a/tests/test_tools.py b/tests/test_tools.py index 93c95ba39..e086bf0de 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -11,6 +11,7 @@ import llm from llm import CancelToolCall, cli +from llm.logs import LogStore from llm.migrations import migrate from llm.tools import llm_time @@ -44,13 +45,10 @@ def multiply(a: int, b: int) -> int: db = sqlite_utils.Database(memory=True) migrate(db) chain_response.log_to_db(db) - assert set(db.table_names()).issuperset( - {"tools", "tool_responses", "tool_calls", "tool_results"} - ) - responses = list(db["responses"].rows) - assert len(responses) == 2 - first_response, second_response = responses + turns = list(db["turns"].rows) + assert len(turns) == 2 + first_turn, second_turn = turns tools = list(db["tools"].rows) assert len(tools) == 1 @@ -58,18 +56,30 @@ def multiply(a: int, b: int) -> int: assert tools[0]["description"] == "Multiply two numbers." assert tools[0]["plugin"] is None - tool_results = list(db["tool_results"].rows) - tool_calls = list(db["tool_calls"].rows) - + # The tool call is in the first turn's output parts; the result is + # among the second turn's inputs. + store = LogStore(db) + first_chain = store.load_chain(first_turn["tip_message_hash"]) + tool_calls = [ + part + for message in first_chain + for part in message.parts + if isinstance(part, llm.parts.ToolCallPart) + ] assert len(tool_calls) == 1 - assert tool_calls[0]["response_id"] == first_response["id"] - assert tool_calls[0]["name"] == "multiply" - assert tool_calls[0]["arguments"] == '{"a": 1231, "b": 2331}' - - assert len(tool_results) == 1 - assert tool_results[0]["response_id"] == second_response["id"] - assert tool_results[0]["output"] == "2869461" - assert tool_results[0]["tool_call_id"] == tool_calls[0]["tool_call_id"] + assert tool_calls[0].name == "multiply" + assert tool_calls[0].arguments == {"a": 1231, "b": 2331} + + second_inputs = store.load_chain(second_turn["parent_message_hash"]) + tool_results_parts = [ + part + for message in second_inputs + for part in message.parts + if isinstance(part, llm.parts.ToolResultPart) + ] + assert len(tool_results_parts) == 1 + assert tool_results_parts[0].output == "2869461" + assert tool_results_parts[0].tool_call_id == tool_calls[0].tool_call_id @pytest.mark.vcr From 501be504fb3a5732d8ae53c7773ebda0de6d1a21 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 20:32:51 -0700 Subject: [PATCH 144/258] llm logs -q: full-text search over the content-addressed tables A turn_search table derives each turn's searchable text - the literal prompt the user typed and the assistant's text output - with an external-content FTS5 index over it. System prompts, fragment contents, tool activity and reasoning are excluded by construction: the text is derived in SQL from the stored part payloads, where text_ref keeps a fragment reference separate from the literals the user typed, rather than from load_chain, which would resolve the fragment text back in. The turn is the search unit because it is the display unit: llm logs renders turns, so matches need no conversion, ranking stays sharp on bounded documents, and each piece of conversation text is indexed exactly once - the index inherits the store's write-only-what-is-new character. One SQL statement serves the migration backfill and the per-turn refresh in LogStore.log (delete-then-derive, so a re-logged turn converges). Ranking is bm25 with the prompt column weighted 10x the response: what you asked says more about what a turn is about than what came back, but the model's answer stays searchable - it is where the memorable tokens you did not know before asking live. Both columns indexed means the weighting is a query-time policy, changeable without a reindex. Legacy rows are searched through their original responses_fts and interleaved on raw bm25, documented as approximate across the two indexes. This fixes a longstanding sign bug: the old path ordered by rank desc, which with FTS5's negative-is-better scores returned the N weakest matches - test expectations encoding that order are updated to best-first. -l/--latest keeps the query as a pure filter in recency order. Bad FTS5 syntax now raises a clean error with a pointer to the query syntax documentation. Design notes and validation in plans/search-design.md. Co-Authored-By: Claude Fable 5 --- docs/changelog.md | 1 + docs/logging.md | 12 +++- llm/cli.py | 49 +++++++++------ llm/logs.py | 101 ++++++++++++++++++++++++++---- llm/migrations.py | 85 ++++++++++++++++++++++++++ tests/test_llm_logs.py | 135 +++++++++++++++++++++++++++++++++++------ tests/test_migrate.py | 6 ++ 7 files changed, 340 insertions(+), 49 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index f663b15f2..38cb4a98b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,7 @@ ## Unreleased +- `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. Results are most-relevant first, with prompt matches weighted well above response matches; `-l/--latest` switches to most-recent first. Searches also cover history recorded by older versions of LLM through its original `responses_fts` index. A longstanding bug where search results were ordered weakest match first is fixed. See {ref}`logging-search`. [#1562](https://github.com/simonw/llm/pull/1562) - `llm logs` now reads both generations of log tables: history recorded by older versions of llm in the legacy `responses` table is merged into the output alongside new turns, with rows from the dual-write era deduplicated by id. The `-m`, `-c`, `-f`, `-T`, `--tools` and `--schema` filters work across both. `-q/--query` full-text search is temporarily unavailable while search against the new tables is designed. [#1562](https://github.com/simonw/llm/pull/1562) - Logging no longer writes the legacy tables: `Response.log_to_db()` records only the content-addressed message store. The legacy tables are kept in place as read-only history. Raw provider payloads (the old `prompt_json` and `response_json` columns) are no longer persisted - the stored message chain is the record of what was sent and returned. [#1562](https://github.com/simonw/llm/pull/1562) - A response logged outside of a conversation now gets a thread of its own, so responses logged through the Python API can be continued with `llm -c` - the same guarantee the `conversations` table used to provide. [#1562](https://github.com/simonw/llm/pull/1562) diff --git a/docs/logging.md b/docs/logging.md index eb9a928f4..f166e05b1 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -173,17 +173,26 @@ llm logs --cid 01h82n0q9crqtnzmf13gkyxawg ### Searching the logs -You can search the logs for a search term in the `prompt` or the `response` columns. +You can search the logs for a search term across your prompts and the model's responses. ```bash llm logs -q 'cheesecake' ``` The most relevant results will be shown first. +Search covers the text you typed and the text the model produced, and nothing else. System prompts, {ref}`fragment ` contents, tool calls and their output, and reasoning traces are all excluded from the index - a query only matches words that appeared in a prompt or a response. If a prompt used fragments, the fragment text is not searchable but the question you typed alongside it is. + +Ranking uses [SQLite FTS5](https://www.sqlite.org/fts5.html) relevance scores, with matches in your prompt weighted well above matches in the response - what you asked is usually a stronger signal of what a conversation was about than what came back. The full [FTS5 query syntax](https://www.sqlite.org/fts5.html#full_text_query_syntax) is available, including phrase queries: +```bash +llm logs -q '"pet pelican"' +``` + To switch to sorting with most recent first, add `-l/--latest`. This can be combined with `-n` to limit the number of results shown: ```bash llm logs -q 'cheesecake' -l -n 3 ``` +Search spans both generations of log tables: new conversations are matched through the `turn_search` index over the content-addressed tables, and history recorded by older versions of LLM is matched through its original `responses_fts` index, with the two result sets merged. The relevance scores come from two separate indexes, so the interleaving between very old and new results is approximate. + (logging-filter-id)= ### Filtering past a specific ID @@ -528,6 +537,7 @@ The full schema for these tables appears in {ref}`the SQL schema section ` definitions were available to a turn, referencing the `tools` table. - `turn_fragments` - which fragments a turn was given, with their `kind` (`prompt` or `system`) and order. Provenance lives here rather than on the shared message rows, and this table is what powers `llm logs -f`. +- `turn_search` - the searchable text of each turn: the literal prompt the user typed (fragment content excluded, via the `text_ref` literals described above) and the assistant's text output. An FTS5 index over this table, `turn_search_fts`, is what powers {ref}`llm logs -q `. Derived from the stored parts when a turn is logged; a turn with no prompt or response text, such as a pure tool call, gets no row. (logging-message-store-queries)= diff --git a/llm/cli.py b/llm/cli.py index 9c49ccf96..b27f07fc0 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -8,6 +8,7 @@ import re import readline import shutil +import sqlite3 import sys import textwrap import warnings @@ -1802,31 +1803,39 @@ def logs_list( # Maybe they uninstalled a model, use the -m option as-is model_id = model - if query: - raise click.ClickException( - "-q/--query is not supported against the new log tables yet" - ) - fragment_hashes = [fragment.id() for fragment in resolve_fragments(db, fragments)] schema_id = make_schema_id(schema)[0] if schema else None store = LogStore(db) - rows = merged_log_rows( - store, - count=count if count and count > 0 else None, - model_id=model_id, - thread_id=conversation_id, - fragment_hashes=fragment_hashes, - tool_names=tools, - any_tools=any_tools, - schema_id=schema_id, - id_gt=id_gt, - id_gte=id_gte, - ) + try: + rows = merged_log_rows( + store, + count=count if count and count > 0 else None, + model_id=model_id, + thread_id=conversation_id, + fragment_hashes=fragment_hashes, + tool_names=tools, + any_tools=any_tools, + schema_id=schema_id, + id_gt=id_gt, + id_gte=id_gte, + query=query, + latest=latest, + ) + except sqlite3.OperationalError as ex: + if query: + # Almost certainly FTS5 syntax - unbalanced quotes, stray + # operators and the like + raise click.ClickException( + f"Invalid search query: {ex} - see the FTS5 query syntax " + "documentation at https://sqlite.org/fts5.html#full_text_query_syntax" + ) + raise - # Newest first out of the query, but read chronologically. - if not data: + # Newest first out of the query, but read chronologically - except + # for search results, which are already most-relevant first. + if not query and not data: rows.reverse() # Attachments, fragments and tool info. New rows carry theirs in @@ -1865,6 +1874,8 @@ def logs_list( "_output_parts", "_parent_message_hash", "_tip_message_hash", + "_legacy", + "_search_rank", ): row.pop(internal, None) diff --git a/llm/logs.py b/llm/logs.py index 2c577bada..09ac71dae 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -21,7 +21,7 @@ import json from typing import Any -from .migrations import migrate +from .migrations import TURN_SEARCH_INSERT_SQL, migrate from .models import Attachment, _conversation_name from .parts import ( AttachmentPart, @@ -456,6 +456,16 @@ def log(self, response, thread_id: str | None = None) -> str: }, replace=True, ) + # Refresh this turn's search row. Delete-then-derive rather than + # replace, so a re-logged turn with a different tip converges on + # what the turn now says. Derived in SQL from the stored + # payloads, not from load_chain - resolving references would put + # fragment content back into the searchable text. + self.db["turn_search"].delete_where("turn_id = ?", [turn_id]) + self.db.execute( + TURN_SEARCH_INSERT_SQL.format(turn_filter="and turns.id = :turn_id"), + {"turn_id": turn_id}, + ) if thread_id is not None: self.db["threads"].update(thread_id, {"tip_message_hash": tip}) return turn_id @@ -669,14 +679,21 @@ def _now() -> str: turns.tip_message_hash, threads.name as conversation_name, turns.model as conversation_model, - schemas.content as schema_json + schemas.content as schema_json{rank_select} from turns left join threads on turns.thread_id = threads.id -left join schemas on turns.schema_id = schemas.id +left join schemas on turns.schema_id = schemas.id{join} {where} -order by turns.id desc{limit} +order by {order_by}{limit} """ +# Relevance ranking for -q. bm25 scores are negative-better, ascending +# order is best-first. The prompt column is weighted well above the +# response: what you typed is a stronger signal of what a turn is about +# than what the model said back. +TURN_SEARCH_RANK = "bm25(turn_search_fts, 10.0, 1.0)" +LEGACY_SEARCH_RANK = "bm25(responses_fts, 10.0, 1.0)" + def _text_of(parts, kind) -> str: "Concatenated text of every part of ``kind`` in order." @@ -722,6 +739,8 @@ def build(self, row: dict) -> dict: "schema_json", ) } + if "_search_rank" in row: + built["_search_rank"] = row["_search_rank"] built.update( { # The turn stores null when no options were set; the @@ -761,11 +780,17 @@ def log_rows( schema_id: str | None = None, id_gt: str | None = None, id_gte: str | None = None, + query: str | None = None, + latest: bool = False, ) -> list[dict]: """Rows for `llm logs`, newest first, drawn from the new tables. Sees only conversations with turns - merged_log_rows adds the rows that exist solely in the legacy `responses` table. + + With ``query`` the rows are the best matches from the turn_search + index, most relevant first - or newest first when ``latest`` is + also set. """ where: list[str] = [] params: dict[str, Any] = {} @@ -807,8 +832,25 @@ def log_rows( where.append(_tool_result_clause(f"and parts.tool_name = :{key}")) params[key] = tool_name + rank_select = "" + join = "" + order_by = "turns.id desc" + if query: + rank_select = f",\n {TURN_SEARCH_RANK} as _search_rank" + join = ( + "\njoin turn_search on turn_search.turn_id = turns.id" + "\njoin turn_search_fts on turn_search_fts.rowid = turn_search.id" + ) + where.append("turn_search_fts match :query") + params["query"] = query + if not latest: + order_by = TURN_SEARCH_RANK + sql = LOG_ROWS_SQL.format( + rank_select=rank_select, + join=join, where=("where " + " and ".join(where)) if where else "", + order_by=order_by, limit=f" limit {count}" if count else "", ) builder = _LogRowBuilder(store) @@ -972,12 +1014,12 @@ def _tool_ids_by_name(store: "LogStore") -> dict: responses.token_details, conversations.name as conversation_name, conversations.model as conversation_model, - schemas.content as schema_json + schemas.content as schema_json{rank_select} from responses left join schemas on responses.schema_id = schemas.id -left join conversations on responses.conversation_id = conversations.id +left join conversations on responses.conversation_id = conversations.id{join} where responses.id not in (select id from turns){extra_where} -order by responses.id desc{limit} +order by {order_by}{limit} """ @@ -993,6 +1035,8 @@ def legacy_log_rows( schema_id: str | None = None, id_gt: str | None = None, id_gte: str | None = None, + query: str | None = None, + latest: bool = False, ) -> list[dict]: """Rows for `llm logs` that exist only in the legacy tables. @@ -1000,6 +1044,9 @@ def legacy_log_rows( schema. Thread ids are conversation ids, so the two filters match the same conversations on either side of the upgrade. """ + if query and "responses_fts" not in db.table_names(): + # A database that never saw legacy llm has no legacy index. + return [] where: list[str] = [] params: dict[str, Any] = {} @@ -1051,8 +1098,22 @@ def legacy_log_rows( )""") params[key] = tool_name + rank_select = "" + join = "" + order_by = "responses.id desc" + if query: + rank_select = f",\n {LEGACY_SEARCH_RANK} as _search_rank" + join = "\njoin responses_fts on responses_fts.rowid = responses.rowid" + where.append("responses_fts match :query") + params["query"] = query + if not latest: + order_by = LEGACY_SEARCH_RANK + sql = LEGACY_LOG_ROWS_SQL.format( + rank_select=rank_select, + join=join, extra_where=(" and " + " and ".join(where)) if where else "", + order_by=order_by, limit=f" limit {count}" if count else "", ) rows = [dict(row) for row in db.query(sql, params)] @@ -1061,17 +1122,35 @@ def legacy_log_rows( return rows -def merged_log_rows(store: "LogStore", *, count: int | None = None, **filters): +def merged_log_rows( + store: "LogStore", + *, + count: int | None = None, + query: str | None = None, + latest: bool = False, + **filters, +): """Rows for `llm logs`: the new tables plus legacy-only responses. Both sides are newest-first and ids are ULIDs on both sides of the upgrade, so a straight sort interleaves the two histories chronologically and the top ``count`` of the union is always within the top ``count`` of each side. + + With ``query``, each side returns its best matches and the union is + ordered most-relevant first. The two bm25 scores come from separate + indexes so the interleave is approximate - close enough in practice, + since both index the same kind of corpus with the same tokenizer. + ``latest`` makes the query a pure filter and keeps recency order. """ - rows = log_rows(store, count=count, **filters) - rows.extend(legacy_log_rows(store.db, count=count, **filters)) - rows.sort(key=lambda row: row["id"], reverse=True) + rows = log_rows(store, count=count, query=query, latest=latest, **filters) + rows.extend( + legacy_log_rows(store.db, count=count, query=query, latest=latest, **filters) + ) + if query and not latest: + rows.sort(key=lambda row: row["_search_rank"]) + else: + rows.sort(key=lambda row: row["id"], reverse=True) if count: rows = rows[:count] return rows diff --git a/llm/migrations.py b/llm/migrations.py index e2b1c3408..ce6d9206b 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -717,3 +717,88 @@ def m024_message_store_payloads(db): ), ) db["turn_fragments"].create_index(["fragment_id"]) + + +# Literal text of one parts row: plain text payloads as-is, text_ref +# payloads contribute only their literal segments - fragment content is +# deliberately not searchable. +TURN_SEARCH_LITERAL = """coalesce( + json_extract(parts.payload, '$.text'), + (select group_concat(json_extract(je.value, '$.literal'), '') + from json_each(parts.payload, '$.text_ref') je + where json_extract(je.value, '$.literal') is not null) +)""" + +# Derives the searchable prompt and response text for turns. Serves both +# the migration backfill (turn_filter="") and the per-turn refresh in +# LogStore.log (turn_filter="and turns.id = :turn_id" - the slot appears +# in three places so the filtered form touches only that turn's chain). +TURN_SEARCH_INSERT_SQL = """ +with recursive output_messages(turn_id, hash) as ( + select turns.id, turns.tip_message_hash + from turns + where turns.tip_message_hash is not null + and (turns.parent_message_hash is null + or turns.tip_message_hash != turns.parent_message_hash) + {turn_filter} + union all + select om.turn_id, messages.parent_hash + from output_messages om + join messages on messages.hash = om.hash + join turns on turns.id = om.turn_id + where messages.parent_hash is not null + and (turns.parent_message_hash is null + or messages.parent_hash != turns.parent_message_hash) +), +prompt_text as ( + select turns.id as turn_id, + (select group_concat({LITERAL}, '') + from parts + where parts.message_hash = turns.parent_message_hash + and parts.type = 'text' + order by parts.position) as text + from turns + join messages on messages.hash = turns.parent_message_hash + where messages.role = 'user' {turn_filter} +), +response_text as ( + select om.turn_id, group_concat(part_text.text, '') as text + from output_messages om + join messages on messages.hash = om.hash and messages.role = 'assistant' + join ( + select parts.message_hash, parts.position, {LITERAL} as text + from parts where parts.type = 'text' + ) part_text on part_text.message_hash = om.hash + group by om.turn_id +) +insert into turn_search (turn_id, prompt, response) +select turns.id, + coalesce(prompt_text.text, ''), + coalesce(response_text.text, '') + from turns + left join prompt_text on prompt_text.turn_id = turns.id + left join response_text on response_text.turn_id = turns.id + where (coalesce(prompt_text.text, '') != '' + or coalesce(response_text.text, '') != '') {turn_filter} +""".replace("{LITERAL}", TURN_SEARCH_LITERAL) + + +@migration +def m025_turn_search(db): + # Searchable text per turn: the user's typed prompt (fragment + # content excluded) and the assistant's text output. An explicit id + # primary key because external-content FTS is keyed by rowid, and + # implicit rowids are not stable across VACUUM. + db["turn_search"].create( + { + "id": int, + "turn_id": str, + "prompt": str, + "response": str, + }, + pk="id", + foreign_keys=(("turn_id", "turns", "id"),), + ) + db["turn_search"].create_index(["turn_id"], unique=True) + db["turn_search"].enable_fts(["prompt", "response"], create_triggers=True) + db.execute(TURN_SEARCH_INSERT_SQL.format(turn_filter="")) diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index 9b939aaca..11104dfda 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -17,13 +17,6 @@ from llm.migrations import migrate from llm.utils import monotonic_ulid -# -q/--query was backed by responses_fts, which is legacy-only. Full -# text search against the new tables has not been designed yet. -search_not_supported = pytest.mark.xfail( - reason="-q/--query has no implementation against the new log tables", -) - - SINGLE_ID = "5843577700ba729bb14c327b30441885" MULTI_ID = "4860edd987df587d042a9eb2b299ce5c" @@ -368,24 +361,130 @@ def test_logs_filtered(user_path, model, path_option): assert all(record["model"] == model for record in records) +def test_logs_search_new_tables(mock_model, logs_db): + runner = CliRunner() + mock_model.enqueue(["A fine city"]) + runner.invoke( + cli, ["-m", "mock", "tell me about Ljubljana"], catch_exceptions=False + ) + mock_model.enqueue(["Try httpx-retries for that"]) + runner.invoke( + cli, ["-m", "mock", "what retry library should I use"], catch_exceptions=False + ) + + # Matches prompt text + result = runner.invoke( + cli, ["logs", "-q", "Ljubljana", "--json"], catch_exceptions=False + ) + assert result.exit_code == 0 + rows = json.loads(result.output) + assert [row["prompt"] for row in rows] == ["tell me about Ljubljana"] + + # Matches response text + result2 = runner.invoke( + cli, ["logs", "-q", "httpx", "--json"], catch_exceptions=False + ) + assert result2.exit_code == 0 + rows2 = json.loads(result2.output) + assert [row["response"] for row in rows2] == ["Try httpx-retries for that"] + + +def test_logs_search_prompt_outranks_response(mock_model, logs_db): + # The prompt column carries a much higher bm25 weight: what you + # typed says more about what a turn is about than what came back. + runner = CliRunner() + mock_model.enqueue(["I recommend a python one-liner"]) + runner.invoke(cli, ["-m", "mock", "how do I sort a list"], catch_exceptions=False) + mock_model.enqueue(["Use yield"]) + runner.invoke( + cli, ["-m", "mock", "explain python generators"], catch_exceptions=False + ) + result = runner.invoke( + cli, ["logs", "-q", "python", "--json"], catch_exceptions=False + ) + assert result.exit_code == 0 + rows = json.loads(result.output) + assert [row["prompt"] for row in rows] == [ + "explain python generators", + "how do I sort a list", + ] + + +def test_logs_search_excludes_fragment_content(mock_model, logs_db, tmpdir): + fragment_path = str(tmpdir / "notes.txt") + with open(fragment_path, "w", encoding="utf-8") as fp: + fp.write("Wombats are sturdy quadrupedal marsupials") + runner = CliRunner() + mock_model.enqueue(["They do indeed"]) + runner.invoke( + cli, + ["-m", "mock", "-f", fragment_path, "do zebras have stripes"], + catch_exceptions=False, + ) + # The typed question is searchable + found = runner.invoke( + cli, ["logs", "-q", "zebras", "--json"], catch_exceptions=False + ) + assert len(json.loads(found.output)) == 1 + # The fragment's content is not + not_found = runner.invoke( + cli, ["logs", "-q", "wombats", "--json"], catch_exceptions=False + ) + assert json.loads(not_found.output) == [] + + +def test_logs_search_merges_legacy_rows(mock_model, logs_db): + # A legacy-only row and a new turn matching the same query both + # come back from one search. + migrate(logs_db) + logs_db["responses"].insert( + { + "id": "01aaaaaaaaaaaaaaaaaaaaaaaa", + "system": None, + "prompt": "name a pet pelican", + "response": "Percy", + "model": "davinci", + "datetime_utc": "2025-01-01T00:00:00", + }, + alter=True, + ) + runner = CliRunner() + mock_model.enqueue(["Scoop"]) + runner.invoke( + cli, ["-m", "mock", "another pelican name please"], catch_exceptions=False + ) + result = runner.invoke( + cli, ["logs", "-q", "pelican", "--json"], catch_exceptions=False + ) + assert result.exit_code == 0 + prompts = {row["prompt"] for row in json.loads(result.output)} + assert prompts == {"name a pet pelican", "another pelican name please"} + + +def test_logs_search_bad_query_is_a_clean_error(logs_db): + runner = CliRunner() + result = runner.invoke(cli, ["logs", "-q", 'unbalanced"quote']) + assert result.exit_code == 1 + assert "Invalid search query" in result.output + + @pytest.mark.parametrize( "query,extra_args,expected", ( # With no search term order should be by datetime ("", [], ["doc1", "doc2", "doc3"]), - # With a search it's order by rank instead - pytest.param("llama", [], ["doc1", "doc3"], marks=search_not_supported), - pytest.param("alpaca", [], ["doc2"], marks=search_not_supported), + # With a search it's order by rank instead - best match first. + # doc3 says llama twice. (The old implementation ordered by + # `rank desc`, which with bm25's negative-is-better scores put + # the weakest matches first; that sign bug is fixed.) + ("llama", [], ["doc3", "doc1"]), + ("alpaca", [], ["doc2"]), # Model filter should work too - pytest.param( - "llama", ["-m", "davinci"], ["doc1", "doc3"], marks=search_not_supported - ), - pytest.param("llama", ["-m", "davinci2"], [], marks=search_not_supported), + ("llama", ["-m", "davinci"], ["doc3", "doc1"]), + ("llama", ["-m", "davinci2"], []), # Adding -l/--latest should return latest first (order by id desc) - pytest.param("llama", ["-l"], ["doc3", "doc1"], marks=search_not_supported), - pytest.param( - "llama", ["--latest"], ["doc3", "doc1"], marks=search_not_supported - ), + ("llama", ["-l"], ["doc3", "doc1"]), + ("llama", ["--latest"], ["doc3", "doc1"]), ), ) def test_logs_search(user_path, query, extra_args, expected): diff --git a/tests/test_migrate.py b/tests/test_migrate.py index a6037a155..330f51a12 100644 --- a/tests/test_migrate.py +++ b/tests/test_migrate.py @@ -50,6 +50,9 @@ def test_migrate_blank(): "responses_ai", "responses_ad", "responses_au", + "turn_search_ai", + "turn_search_ad", + "turn_search_au", } @@ -90,6 +93,9 @@ def test_migrate_from_original_schema(has_record): "responses_ai", "responses_ad", "responses_au", + "turn_search_ai", + "turn_search_ad", + "turn_search_au", } From 7090c1a0f2127a90bf4c9d85766c9659eaaf8cc0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 20:49:36 -0700 Subject: [PATCH 145/258] Cleanup: cog pins to sqlite-utils 4.1.1, drop conversation-views.sql - The cog check pinned sqlite-utils inconsistently (4.0rc2 in the Justfile, 4.0rc4 in CI); both now use the 4.1.1 release. Regenerating under 4.1.1 produced no changes, so the pins were only drifting, not disagreeing. - conversation-views.sql was a hand-applied prototype; the documented cookbook query in docs/logging.md replaced it. - The fork-diagram placeholder in the message store docs is removed - the prose stands on its own. Co-Authored-By: Claude Fable 5 --- .github/workflows/test.yml | 4 +- Justfile | 4 +- conversation-views.sql | 152 ------------------------------------- docs/logging.md | 2 - 4 files changed, 4 insertions(+), 158 deletions(-) delete mode 100644 conversation-views.sql diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e3ab350af..888e4a543 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: include: - os: ubuntu-latest python-version: "3.14" - sqlite-utils-version: "4.0rc4" + sqlite-utils-version: "4.1.1" steps: - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} @@ -36,7 +36,7 @@ jobs: run: | python -m pytest -vv - name: Check if cog needs to be run - if: matrix.sqlite-utils-version == '4.0rc4' + if: matrix.sqlite-utils-version == '4.1.1' run: | cog --check \ -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" \ diff --git a/Justfile b/Justfile index c74742bca..643c887fe 100644 --- a/Justfile +++ b/Justfile @@ -11,7 +11,7 @@ echo " Black" uv run black . --check echo " cog" - uv run --with sqlite-utils==4.0rc2 cog --check \ + uv run --with sqlite-utils==4.1.1 cog --check \ -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" \ README.md docs/*.md echo " mypy" @@ -25,7 +25,7 @@ # Rebuild docs with cog @cog: - uv run --with sqlite-utils==4.0rc2 cog -r -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" docs/**/*.md docs/*.md README.md + uv run --with sqlite-utils==4.1.1 cog -r -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" docs/**/*.md docs/*.md README.md # Serve live docs on localhost:8000 @docs: cog diff --git a/conversation-views.sql b/conversation-views.sql deleted file mode 100644 index bec9d595d..000000000 --- a/conversation-views.sql +++ /dev/null @@ -1,152 +0,0 @@ --- A readable rendering of the content-addressed message tree. --- --- Prototype: applied directly to logs.db rather than added to --- llm/migrations.py, so it can be reshaped without a migration. --- Re-runnable - the view is dropped first. --- --- select entry from conversation_tree; -- everything --- select entry from conversation_tree where id = 'faafcc3b'; --- --- One row per part, depth-first, so reading `entry` top to bottom --- replays the conversation. Indentation tracks *branching*, not depth: --- a conversation that never forks stays flush left however long it --- runs, and each divergence steps one level in. Where a message forked, --- each branch is headed [n/total] and its whole subtree is aligned --- underneath, so it is always clear which reply belongs to which try. - -DROP VIEW IF EXISTS conversation_tree; -DROP VIEW IF EXISTS part_text; - --- Resolves a part's text back from storage. Text that borrowed from a --- fragment is stored as an ordered list of fragment references and --- literals rather than a copy, so reading it means splicing the --- fragment contents back in. -CREATE VIEW part_text AS -SELECT - p.id AS part_id, - coalesce( - json_extract(p.payload, '$.text'), - ( - SELECT group_concat( - coalesce( - json_extract(piece.value, '$.literal'), - (SELECT f.content FROM fragments f - WHERE f.id = json_extract(piece.value, '$.fragment')) - ), - '' ORDER BY piece.key - ) - FROM json_each(json_extract(p.payload, '$.text_ref')) piece - ) - ) AS text -FROM parts p; - -CREATE VIEW conversation_tree AS -WITH RECURSIVE --- Where each message sits among its siblings. Computed up front --- because window functions are not allowed in a recursive term. -sibling AS ( - SELECT - hash, - parent_hash, - row_number() OVER (PARTITION BY parent_hash ORDER BY rowid) AS ord, - count(*) OVER (PARTITION BY parent_hash) AS of - FROM messages -), -walk(root_hash, message_hash, depth, indent, ord, of, sort_key) AS ( - SELECT m.hash, m.hash, 0, 0, 1, 1, printf('%08d', m.rowid) - FROM messages m - WHERE m.parent_hash IS NULL - UNION ALL - SELECT - w.root_hash, - s.hash, - w.depth + 1, - -- Step in only where the parent actually forked. - w.indent + (CASE WHEN s.of > 1 THEN 1 ELSE 0 END), - s.ord, - s.of, - w.sort_key || '/' || printf('%08d', m.rowid) - FROM sibling s - JOIN messages m ON m.hash = s.hash - JOIN walk w ON s.parent_hash = w.message_hash -), -rendered AS ( - SELECT - w.*, - m.role, - p.position, - p.type, - -- Prefix: role for plain text, role + kind for anything else, so - -- a reasoning block or a tool call is never mistaken for what - -- the model actually said. - m.role - || CASE - WHEN p.type IS NULL OR p.type = 'text' THEN '' - ELSE ' ' || p.type - END - || ': ' AS prefix, - CASE p.type - WHEN 'text' THEN coalesce(pt.text, '') - WHEN 'reasoning' THEN - CASE - WHEN json_extract(p.payload, '$.redacted') - THEN '(reasoning withheld by provider)' - ELSE coalesce(pt.text, '') - END - WHEN 'tool_call' THEN - p.tool_name - || '(' || coalesce(json_extract(p.payload, '$.arguments'), '') || ')' - WHEN 'tool_result' THEN - p.tool_name - || ' -> ' || coalesce(json_extract(p.payload, '$.output'), '') - WHEN 'attachment' THEN '(attachment)' - ELSE coalesce(p.type, '(no content)') - END AS body - FROM walk w - JOIN messages m ON m.hash = w.message_hash - LEFT JOIN parts p ON p.message_hash = m.hash - LEFT JOIN part_text pt ON pt.part_id = p.id -), -margined AS ( - SELECT - r.*, - CASE - WHEN r.indent = 0 THEN '' - ELSE replace(hex(zeroblob((r.indent - 1) * 8)), '00', ' ') - -- The branch marker occupies the last indent step, so - -- the head of a branch and its descendants line up. - || CASE - WHEN r.of > 1 AND coalesce(r.position, 0) = 0 - THEN printf('%-8s', '[' || r.ord || '/' || r.of || ']') - ELSE ' ' - END - END AS margin - FROM rendered r -) -SELECT - -- Short, typeable handle for the whole tree. Every message reachable - -- from one root shares it, so filtering on it gives that - -- conversation and every branch of it. - substr(m.root_hash, 4, 8) AS id, - m.margin - || m.prefix - -- Wrapped lines sit under the prefix, so a multi-line answer - -- stays inside its own column. - || replace( - m.body, - char(10), - char(10) || replace( - hex(zeroblob(length(m.margin) + length(m.prefix))), '00', ' ' - ) - ) AS entry, - m.depth, - m.indent, - CASE WHEN m.of > 1 THEN m.ord ELSE NULL END AS branch, - m.of AS branches, - m.role, - m.type, - m.message_hash, - m.root_hash, - m.sort_key -FROM margined m -ORDER BY m.sort_key, m.position; diff --git a/docs/logging.md b/docs/logging.md index f166e05b1..0c202a528 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -506,8 +506,6 @@ Two conversations of two turns each - eight messages sent to the model in total Because messages form a parent-linked tree, a conversation can fork: a new thread can point at any existing message and continue from there, sharing its entire history with the thread it came from until the two diverge. Nothing is copied when this happens. The `threads.forked_from` column records which thread a fork came from. -TODO: description of diagram - Shared rows cut both ways. Deleting a conversation is not the same as deleting its rows, because another thread may reach the same messages - removing them would silently corrupt that thread's history. For this reason LLM does not currently delete message rows at all; garbage collection of unreachable messages is deliberately left as future work. (logging-message-store-references)= From 41a8291193dc3e2720b69ba6342cb754740f6453 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 21:33:13 -0700 Subject: [PATCH 146/258] Parts storage: literal text in its own column, structure-only payloads {"text": "...", "type": "text"} wrapped every piece of response text in JSON for no reader's benefit: the type key duplicated the type column, and the text paid JSON string escaping on every newline and quote. Literal text now lives in a dedicated text column - raw, unescaped and never parsed, so text that happens to look like JSON is safe and `select text from parts` reads as prose. The payload column keeps only structure: text_ref fragment references, tool call fields, redacted flags, provider metadata - without the type key - and is NULL when the text column carries the whole part. A part is literal or structured by column, never by sniffing content, which is what makes the fragment reference encoding safe where raw-text-with-markers would not be. Storage encoding only: hashes are computed over resolved message content before writing, so no message hash changes. m027 rewrites existing rows; m025's search backfill gains a guard adding the column early, for databases migrating from m024 straight through (the turn_search SQL now reads parts.text). The verify() tamper test moves its tampering to the text column, where a plain part's text actually lives. Co-Authored-By: Claude Fable 5 --- docs/changelog.md | 1 + docs/logging.md | 28 +++++++++++------- llm/logs.py | 39 ++++++++++++++++++++----- llm/migrations.py | 48 +++++++++++++++++++++++++----- tests/test_logs_store.py | 63 +++++++++++++++++++++++++++++++++++++--- 5 files changed, 149 insertions(+), 30 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 38cb4a98b..777a9dff5 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,7 @@ ## Unreleased +- The `parts` table stores literal text in a dedicated `text` column - raw and unescaped, so prose reads as prose in SQL queries - instead of wrapping it in JSON. The `payload` column now holds only structure (fragment references, tool call fields, provider metadata), without the redundant `type` key, and is NULL when the text column carries the whole part. Storage encoding only: message hashes are computed over resolved content and are unchanged. Existing databases are rewritten by a migration. [#1562](https://github.com/simonw/llm/pull/1562) - `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. Results are most-relevant first, with prompt matches weighted well above response matches; `-l/--latest` switches to most-recent first. Searches also cover history recorded by older versions of LLM through its original `responses_fts` index. A longstanding bug where search results were ordered weakest match first is fixed. See {ref}`logging-search`. [#1562](https://github.com/simonw/llm/pull/1562) - `llm logs` now reads both generations of log tables: history recorded by older versions of llm in the legacy `responses` table is merged into the output alongside new turns, with rows from the dual-write era deduplicated by id. The `-m`, `-c`, `-f`, `-T`, `--tools` and `--schema` filters work across both. `-q/--query` full-text search is temporarily unavailable while search against the new tables is designed. [#1562](https://github.com/simonw/llm/pull/1562) - Logging no longer writes the legacy tables: `Response.log_to_db()` records only the content-addressed message store. The legacy tables are kept in place as read-only history. Raw provider payloads (the old `prompt_json` and `response_json` columns) are no longer persisted - the stored message chain is the record of what was sent and returned. [#1562](https://github.com/simonw/llm/pull/1562) diff --git a/docs/logging.md b/docs/logging.md index 0c202a528..2e92518e6 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -352,7 +352,14 @@ for row in db.query("select * from messages"): "select * from parts where message_hash = ? order by position", [row["hash"]], ): - lines.append(" part {}: {}".format(part["position"], part["payload"])) + lines.append( + " part {}: type={} text={!r} payload={}".format( + part["position"], + part["type"], + part["text"], + part["payload"] or "null", + ) + ) lines.append("") lines.append("turns:") for row in db.query("select * from turns order by id"): @@ -375,19 +382,19 @@ messages and their parts: user b2:d6b0cd4e7a65ea90423c50fadb3f5704 parent: null - part 0: {"type": "text", "text": "Suggest a name for a pet pelican"} + part 0: type=text text='Suggest a name for a pet pelican' payload=null assistant b2:0f2c02ad982050b623b7e034199c8c61 parent: b2:d6b0cd4e7a65ea90423c50fadb3f5704 - part 0: {"type": "text", "text": "How about Percy? Pelicans suit a dignified name."} + part 0: type=text text='How about Percy? Pelicans suit a dignified name.' payload=null user b2:c785dd6c77540150c2647f406cacc76f parent: b2:0f2c02ad982050b623b7e034199c8c61 - part 0: {"type": "text", "text": "Now one for a pet walrus"} + part 0: type=text text='Now one for a pet walrus' payload=null assistant b2:a30a236e0d1b717c592e826d06e3c9d2 parent: b2:c785dd6c77540150c2647f406cacc76f - part 0: {"type": "text", "text": "Wallace. It pairs nicely with Percy."} + part 0: type=text text='Wallace. It pairs nicely with Percy.' payload=null turns: @@ -414,7 +421,7 @@ thread $THREAD_ID Things to notice: - The four messages form a chain: the first has a `null` parent and each subsequent message names the hash of the one before it. -- The `part N:` lines show each part's `payload` column verbatim - the part serialized as JSON. +- The `part N:` lines show each part's storage columns. A part whose text is pure literal keeps it in the `text` column - raw, unescaped and never parsed, so text that happens to look like JSON is safe - with a `null` payload. The `payload` column holds any remaining structure as JSON: fragment references, tool call fields, provider metadata. The part's type lives only in the `type` column. - Each turn brackets one model call. Its `parent_message_hash` is the tip of the chain that was sent to the model - the last input message, usually that turn's user prompt - and its `tip_message_hash` is the chain tip after the model's reply was appended. The prompt and response that `llm logs` displays are derived by splitting the chain at the parent, rather than being stored a second time. - The thread's id is the conversation id, its name is derived from the first prompt and its `tip_message_hash` follows the head of the conversation as new turns are logged. - The ULID identifiers - sortable random identifiers issued in time order, the same id space older versions used for response ids - and the timestamp columns have been replaced with placeholders because they change on every run. The hashes have not: they depend only on the message content and its position in the chain, so replaying this conversation produces these exact four hashes. @@ -512,10 +519,10 @@ Shared rows cut both ways. Deleting a conversation is not the same as deleting i ### Storage by reference -The hash of a message covers its resolved content, but storage is by reference. A text part whose content borrows from a fragment does not store a copy of that fragment. Instead of a `text` key its payload holds a `text_ref` list of fragment references and literal segments: +The hash of a message covers its resolved content, but storage is by reference. A text part whose content borrows from a fragment does not store a copy of that fragment. Instead of a filled `text` column its payload holds a `text_ref` list of fragment references and literal segments: ```json -{"type": "text", "text_ref": [{"fragment": 1}, {"literal": "\nquestion about it"}]} +{"text_ref": [{"fragment": 1}, {"literal": "\nquestion about it"}]} ``` Here fragment `1` is an id in the existing `fragments` table. Reading the part concatenates the fragment content and the literal back together, reproducing the exact text that was hashed. Ask a hundred questions about a novel and the novel is stored once. @@ -529,7 +536,7 @@ Attachments work the same way: the binary content lives in the `attachments` tab The full schema for these tables appears in {ref}`the SQL schema section ` below. - `messages` - one row per unique message. `hash` is the content address described above, `parent_hash` links to the previous message in the chain and `role` is `system`, `user` or `assistant`. `provider_metadata` holds any provider-specific data carried by the message; it participates in the hash. -- `parts` - the content of each message, ordered by `position`. `payload` is the part serialized as JSON - stored in the order the model produced it, not the canonical key-sorted form used for hashing. `type` and `tool_name` are copied out of the payload so they can be filtered on directly. +- `parts` - the content of each message, ordered by `position`. `text` holds the part's literal text when it borrows no fragments - raw and never parsed, so `select text from parts` reads as prose. `payload` holds whatever structure remains as JSON (fragment references, tool call fields, provider metadata), in the order the model produced it, or NULL when the text column carries the whole part. `type` and `tool_name` are their own columns for direct filtering; the type never appears inside the payload. - `part_attachments` and `part_fragments` - junction tables recording which rows in `attachments` and `fragments` a part's payload references, in order. - `threads` - one row per conversation. `id` is the conversation id, `tip_message_hash` points at the current head of the conversation and `forked_from` records the thread a fork came from. - `turns` - one row per model call. `parent_message_hash` and `tip_message_hash` bracket the call's input and output as described above, and the remaining columns record provenance: model, options, schema, token counts and timings. Turn ids are ULIDs, in the same id space as legacy response ids, which is how the two generations of tables sort together in `llm logs`. @@ -572,7 +579,7 @@ select substr(' ', 1, walk.indent * 4) || messages.role || ': ' || coalesce( - json_extract(parts.payload, '$.text'), + parts.text, ( select group_concat( coalesce( @@ -803,6 +810,7 @@ CREATE TABLE "parts" ( "position" INTEGER, "type" TEXT, "tool_name" TEXT, + "text" TEXT, "payload" TEXT ); CREATE TABLE "part_attachments" ( diff --git a/llm/logs.py b/llm/logs.py index f4b405641..d74892962 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -169,6 +169,9 @@ def _write_part( fragment_map: dict[str, int], ) -> None: payload = part.to_dict() + # The type key is redundant with the type column; readers put it + # back from there. + part_type = payload.pop("type") attachments = _attachments_of(part) # Large content out of the payload and into the tables that @@ -179,23 +182,32 @@ def _write_part( attachment_ids = [ ensure_attachment(self.db, attachment) for attachment in attachments ] - _encode_attachment_refs(payload, attachment_ids) + _encode_attachment_refs(payload, attachment_ids, part_type) else: attachment_ids = [] + # Pure literal text lives in its own column - raw, unescaped, + # never parsed - so prose reads as prose in SQL. Text that + # borrows fragments stays structured in the payload as text_ref. + text = None + if part_type in ("text", "reasoning") and "text" in payload: + text = payload.pop("text") + part_id = ( self.db["parts"] .insert( { "message_hash": message_hash_, "position": position, - "type": payload["type"], + "type": part_type, "tool_name": payload.get("name"), + "text": text, # Plain dumps, not canonical_json: sorting keys is # for hashing. Storage keeps the order the model # produced, so tool call arguments read back as - # they were written. - "payload": json.dumps(payload), + # they were written. NULL when the text column + # carries the whole part. + "payload": json.dumps(payload) if payload else None, } ) .last_pk @@ -259,7 +271,16 @@ def _load_parts(self, message_hashes: list[str]) -> dict[str, list[Any]]: message_hashes, ) ) - payloads = [json.loads(row["payload"]) for row in part_rows] + # Rebuild each part's full dict from its columns: type from the + # type column, literal text from the text column, everything + # else from the JSON payload. + payloads: list[Any] = [] + for row in part_rows: + payload = json.loads(row["payload"]) if row["payload"] else {} + payload["type"] = row["type"] + if row["text"] is not None: + payload["text"] = row["text"] + payloads.append(payload) # Resolve the references put in on the way in. Both lookups are # batched across the whole chain rather than done per part. fragments = self._load_fragments(payloads) @@ -614,11 +635,13 @@ def _fragment_ids(payload: dict) -> list[int]: ] -def _encode_attachment_refs(payload: dict, attachment_ids: list[str]) -> None: +def _encode_attachment_refs( + payload: dict, attachment_ids: list[str], part_type: str +) -> None: "Replace inline attachment dicts with their content-addressed ids." - if payload["type"] == "attachment": + if part_type == "attachment": payload["attachment"] = {"id": attachment_ids[0]} - elif payload["type"] == "tool_result": + elif part_type == "tool_result": payload["attachments"] = [{"id": id} for id in attachment_ids] diff --git a/llm/migrations.py b/llm/migrations.py index ce6d9206b..f2c5f8837 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -1,4 +1,5 @@ import datetime +import json from collections.abc import Callable MIGRATIONS: list[Callable] = [] @@ -605,10 +606,14 @@ def m024_message_store_payloads(db): "type": str, # Tool name for tool_call and tool_result parts, else NULL. "tool_name": str, - # Part.to_dict(), with large content replaced by references. - # Authoritative: reading is Part.from_dict(resolved payload). - # type and tool_name above are write-time projections for - # querying; the read path ignores them. + # The part's literal text, for text and reasoning parts whose + # text borrows no fragments. Raw and unescaped - this column + # is never parsed as anything. + "text": str, + # The part's remaining structure as JSON, with large content + # replaced by references (fragment ids for text, attachment + # ids for binary) and the type key left to the column above. + # NULL when the text column carries everything. "payload": str, }, pk="id", @@ -719,11 +724,11 @@ def m024_message_store_payloads(db): db["turn_fragments"].create_index(["fragment_id"]) -# Literal text of one parts row: plain text payloads as-is, text_ref -# payloads contribute only their literal segments - fragment content is -# deliberately not searchable. +# Literal text of one parts row: the text column when the part's text +# is pure literal, otherwise the literal segments of a text_ref payload +# - fragment content is deliberately not searchable. TURN_SEARCH_LITERAL = """coalesce( - json_extract(parts.payload, '$.text'), + parts.text, (select group_concat(json_extract(je.value, '$.literal'), '') from json_each(parts.payload, '$.text_ref') je where json_extract(je.value, '$.literal') is not null) @@ -801,4 +806,31 @@ def m025_turn_search(db): ) db["turn_search"].create_index(["turn_id"], unique=True) db["turn_search"].enable_fts(["prompt", "response"], create_triggers=True) + # The backfill SQL reads parts.text, which m027 introduces - a + # database migrating from m024 straight through needs the column to + # exist before this runs. m027 skips the add when it is present. + if "text" not in db["parts"].columns_dict: + db["parts"].add_column("text", str) db.execute(TURN_SEARCH_INSERT_SQL.format(turn_filter="")) + + +@migration +def m027_parts_text_column(db): + # Literal text moves out of the JSON payload into its own column - + # raw, never escaped, never parsed - and the redundant "type" key + # (already a column) leaves every payload. What structure remains + # is stored as JSON, or NULL when the text column carries the whole + # part. Storage encoding only: hashes are computed over resolved + # message content before anything is written, so no hash changes. + if "text" not in db["parts"].columns_dict: + db["parts"].add_column("text", str) + for row in list(db.query("select id, type, payload from parts")): + payload = json.loads(row["payload"]) if row["payload"] else {} + payload.pop("type", None) + text = None + if row["type"] in ("text", "reasoning") and "text" in payload: + text = payload.pop("text") + db["parts"].update( + row["id"], + {"text": text, "payload": json.dumps(payload) if payload else None}, + ) diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 1bbbe9ae7..2afefb25e 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -786,6 +786,63 @@ def test_successive_library_turns_extend_the_thread(self, store, mock_model): # ---- storage by reference -------------------------------------------- +class TestPartStorageFormat: + """Literal text is stored raw in its own column - never escaped, + never parsed - and the JSON payload holds only structure, with the + type key left to the type column.""" + + def test_plain_text_stores_raw_text_and_no_payload(self, store): + text = '{"looks": "like json", "but": "is text"}' + store.ensure_chain([llm.user(text)]) + row = next(iter(store.db["parts"].rows)) + assert row["type"] == "text" + assert row["text"] == text + assert row["payload"] is None + + def test_redacted_reasoning_splits_text_from_structure(self, store): + message = Message( + role="assistant", + parts=[ReasoningPart(text="thinking", redacted=True)], + ) + tip = store.ensure_chain([message]) + row = next(iter(store.db["parts"].rows)) + assert row["text"] == "thinking" + assert json.loads(row["payload"]) == {"redacted": True} + assert store.load_chain(tip) == [message] + + def test_no_stored_payload_contains_a_type_key(self, store): + messages = [ + llm.user("hi"), + Message( + role="assistant", + parts=[ + ReasoningPart(text="thinking", redacted=True), + ToolCallPart(name="t", arguments={"a": 1}, tool_call_id="c1"), + ], + ), + Message( + role="tool", + parts=[ToolResultPart(name="t", output="ok", tool_call_id="c1")], + ), + ] + tip = store.ensure_chain(messages) + for row in store.db["parts"].rows: + if row["payload"] is not None: + assert "type" not in json.loads(row["payload"]) + assert store.load_chain(tip) == messages + assert store.verify() == [] + + def test_fragment_referencing_text_stays_structured(self, store): + novel = "CALL ME ISHMAEL. " * 20 + ensure_fragment(store.db, novel) + store.ensure_chain([llm.user(f"{novel}\nwho?")], fragments=[novel]) + row = next(iter(store.db["parts"].rows)) + assert row["text"] is None + assert json.loads(row["payload"]) == { + "text_ref": [{"fragment": 1}, {"literal": "\nwho?"}] + } + + class TestFragmentReferences: """The point of fragments is that a novel is stored once and pointed at from every prompt about it, so the text must not be expanded into @@ -914,12 +971,10 @@ def test_every_kind_of_part_verifies(self, store): ) assert store.verify() == [] - def test_a_corrupted_payload_is_caught(self, store): + def test_a_corrupted_part_is_caught(self, store): tip = store.ensure_chain([llm.user("Hi")]) with store.db.conn: - store.db.execute( - "update parts set payload = ?", ['{"type":"text","text":"tampered"}'] - ) + store.db.execute("update parts set text = 'tampered'") assert store.verify() == [tip] def test_a_missing_fragment_is_caught(self, store): From 3e3a1bc6dccdf615f4a73b33cb72c8adbbea82c2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 21:43:55 -0700 Subject: [PATCH 147/258] tool_instantiations: record and display toolbox instance provenance A new tool_instantiations table records which configured toolbox instance served each tool call - name, plugin and constructor arguments, keyed by tool_call_id. Message rows are shared and cannot carry local execution facts, so this joins to the chain from outside it, the same pattern as turn_search; a comment marks it as the seed of a fuller execution-events table. llm logs --json now shows an "instance" object on each tool result - capability the legacy tables recorded but never displayed. Legacy rows gain the same display through their existing tool_instances data, so both generations render identically, and the toolbox tests get their instance assertions back - corrected: the old expectations said the Memory results were served by a "Filesystem" instance, an artifact of a leaked loop variable in the legacy writer. The write path asks the instance for its _config rather than trusting the Toolbox annotation, because a tool built from a bound method carries its __self__ here - a --functions tool made from a builtin arrives with the builtins module as its instance. Also documents the logprobs consequence of dropping raw provider payloads, in the changelog and the OpenAI completion models docs: logprobs are no longer persisted, remain readable at runtime via the Python API, and rows logged by older versions keep their stored values. Co-Authored-By: Claude Fable 5 --- docs/changelog.md | 3 +- docs/logging.md | 1 + docs/openai-models.md | 2 +- llm/logs.py | 64 ++++++++++++++++++++++++++++++++-- llm/migrations.py | 20 +++++++++++ tests/test_logs_store.py | 28 +++++++++++++++ tests/test_plugins.py | 74 ++++++++++++++++++++++++++++++++-------- 7 files changed, 173 insertions(+), 19 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 777a9dff5..d0d9599fa 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,10 +2,11 @@ ## Unreleased +- `llm logs --json` now shows which configured toolbox instance served each tool result - the toolbox name, plugin and constructor arguments, so a `SQLite_query` result records that it ran against `SQLite("mydb.db")`. Recorded in the new `tool_instantiations` table for new logs, and read from the legacy `tool_instances` data for history logged by older versions of LLM, which recorded it without ever displaying it. [#1562](https://github.com/simonw/llm/pull/1562) - The `parts` table stores literal text in a dedicated `text` column - raw and unescaped, so prose reads as prose in SQL queries - instead of wrapping it in JSON. The `payload` column now holds only structure (fragment references, tool call fields, provider metadata), without the redundant `type` key, and is NULL when the text column carries the whole part. Storage encoding only: message hashes are computed over resolved content and are unchanged. Existing databases are rewritten by a migration. [#1562](https://github.com/simonw/llm/pull/1562) - `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. Results are most-relevant first, with prompt matches weighted well above response matches; `-l/--latest` switches to most-recent first. Searches also cover history recorded by older versions of LLM through its original `responses_fts` index. A longstanding bug where search results were ordered weakest match first is fixed. See {ref}`logging-search`. [#1562](https://github.com/simonw/llm/pull/1562) - `llm logs` now reads both generations of log tables: history recorded by older versions of llm in the legacy `responses` table is merged into the output alongside new turns, with rows from the dual-write era deduplicated by id. The `-m`, `-c`, `-f`, `-T`, `--tools` and `--schema` filters work across both. `-q/--query` full-text search is temporarily unavailable while search against the new tables is designed. [#1562](https://github.com/simonw/llm/pull/1562) -- Logging no longer writes the legacy tables: `Response.log_to_db()` records only the content-addressed message store. The legacy tables are kept in place as read-only history. Raw provider payloads (the old `prompt_json` and `response_json` columns) are no longer persisted - the stored message chain is the record of what was sent and returned. [#1562](https://github.com/simonw/llm/pull/1562) +- Logging no longer writes the legacy tables: `Response.log_to_db()` records only the content-addressed message store. The legacy tables are kept in place as read-only history. Raw provider payloads (the old `prompt_json` and `response_json` columns) are no longer persisted - the stored message chain is the record of what was sent and returned. This means data that only ever lived in those raw payloads is no longer stored, notably the log probabilities returned by OpenAI completion models with `-o logprobs` - those remain available on the response object through the Python API, and rows logged by older versions of LLM keep the values they recorded. [#1562](https://github.com/simonw/llm/pull/1562) - A response logged outside of a conversation now gets a thread of its own, so responses logged through the Python API can be continued with `llm -c` - the same guarantee the `conversations` table used to provide. [#1562](https://github.com/simonw/llm/pull/1562) - New documentation for the message store: the schema, the content-addressing hash contract, worked examples and a SQL cookbook, in {ref}`the logging documentation `. [#1562](https://github.com/simonw/llm/pull/1562) - Passing `prompt=`, `fragments=`, `attachments=` or `tool_results=` alongside `messages=` to `model.prompt()` or `conversation.prompt()` now appends that new input to the supplied message history, instead of silently omitting it from `prompt.messages` - previously the model could receive text that never appeared in the logged conversation. [#1562](https://github.com/simonw/llm/pull/1562) diff --git a/docs/logging.md b/docs/logging.md index 2e92518e6..95fa4b712 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -543,6 +543,7 @@ The full schema for these tables appears in {ref}`the SQL schema section ` definitions were available to a turn, referencing the `tools` table. - `turn_fragments` - which fragments a turn was given, with their `kind` (`prompt` or `system`) and order. Provenance lives here rather than on the shared message rows, and this table is what powers `llm logs -f`. - `turn_search` - the searchable text of each turn: the literal prompt the user typed (fragment content excluded, via the `text_ref` literals described above) and the assistant's text output. An FTS5 index over this table, `turn_search_fts`, is what powers {ref}`llm logs -q `. Derived from the stored parts when a turn is logged; a turn with no prompt or response text, such as a pure tool call, gets no row. +- `tool_instantiations` - which configured {ref}`toolbox ` instance served a tool call: the toolbox name, its plugin and its constructor arguments, keyed by `tool_call_id`. Message rows are shared between conversations and so cannot carry this kind of local execution provenance; this table joins to the chain from outside it, and is what lets `llm logs --json` show that a `SQLite_query` call ran against `SQLite("mydb.db")`. (logging-message-store-queries)= diff --git a/docs/openai-models.md b/docs/openai-models.md index c8c33b79b..33b269c52 100644 --- a/docs/openai-models.md +++ b/docs/openai-models.md @@ -123,7 +123,7 @@ The vector size of the supported OpenAI embedding models are as follows: The `gpt-3.5-turbo-instruct` model is a little different - it is a completion model rather than a chat model, described in [the OpenAI completions documentation](https://platform.openai.com/docs/api-reference/completions/create). -Completion models can be called with the `-o logprobs 3` option (not supported by chat models) which will cause LLM to store 3 log probabilities for each returned token in the SQLite database. Consult [this issue](https://github.com/simonw/llm/issues/284#issuecomment-1724772704) for details on how to read these values. +Completion models can be called with the `-o logprobs 3` option (not supported by chat models) which will request 3 log probabilities for each returned token. These are no longer stored in the SQLite log database - raw provider payloads are not persisted there - but they can be read at runtime from `response.response_json` when using the {ref}`Python API `. Rows logged by older versions of LLM retain the values they stored; consult [this issue](https://github.com/simonw/llm/issues/284#issuecomment-1724772704) for how to read those. (openai-extra-models)= diff --git a/llm/logs.py b/llm/logs.py index d74892962..95539e6e3 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -477,6 +477,32 @@ def log(self, response, thread_id: str | None = None) -> str: }, replace=True, ) + # Which configured toolbox instance served each tool call. This + # is local execution provenance, so it lives outside the hashed + # message tree, keyed by the tool_call_id both worlds share. + for tool_result in response.prompt.tool_results: + # instance is annotated as Toolbox but a tool built from a + # bound method can carry an arbitrary __self__ here, so ask + # for the config rather than trusting the type. + config = getattr(tool_result.instance, "_config", None) + if config is None or not tool_result.tool_call_id: + continue + self.db["tool_instantiations"].insert( + { + "tool_call_id": tool_result.tool_call_id, + "name": tool_result.name.split("_")[0], + "plugin": next( + ( + tool.plugin + for tool in response.prompt.tools + if tool.name == tool_result.name + ), + None, + ), + "arguments": json.dumps(config), + }, + replace=True, + ) # Refresh this turn's search row. Delete-then-derive rather than # replace, so a re-logged turn with a different tip converges on # what the turn now says. Derived in SQL from the stored @@ -923,6 +949,12 @@ def log_row_extras(store: "LogStore", row: dict) -> dict: for part in row.get("_output_parts", []) if isinstance(part, ToolCallPart) ] + result_parts = [ + part for part in row.get("_input_parts", []) if isinstance(part, ToolResultPart) + ] + instances = _instances_by_tool_call_id( + store, [part.tool_call_id for part in result_parts if part.tool_call_id] + ) tool_results = [ { "id": result_ids.get(part.tool_call_id), @@ -931,10 +963,10 @@ def log_row_extras(store: "LogStore", row: dict) -> dict: "output": part.output, "tool_call_id": part.tool_call_id, "exception": part.exception, + "instance": instances.get(part.tool_call_id), "attachments": [_attachment_summary(a) for a in part.attachments], } - for part in row.get("_input_parts", []) - if isinstance(part, ToolResultPart) + for part in result_parts ] # input_schema is rendered as a dict, so decode it here rather than @@ -1014,6 +1046,28 @@ def _tool_ids_by_name(store: "LogStore") -> dict: } +def _instances_by_tool_call_id(store: "LogStore", tool_call_ids: list) -> dict: + "Which configured toolbox instance served each call, for display." + if not tool_call_ids: + return {} + placeholders = ",".join("?" * len(tool_call_ids)) + return { + row["tool_call_id"]: { + "name": row["name"], + "plugin": row["plugin"], + "arguments": row["arguments"], + } + for row in store.db.query( + f""" + select tool_call_id, name, plugin, arguments + from tool_instantiations + where tool_call_id in ({placeholders}) + """, + tool_call_ids, + ) + } + + # -- legacy rows --------------------------------------------------------- # # History logged by older versions of llm lives only in the `responses` @@ -1260,6 +1314,11 @@ def merged_log_rows( 'output', tr.output, 'tool_call_id', tr.tool_call_id, 'exception', tr.exception, + 'instance', case when ti.id is not null then json_object( + 'name', ti.name, + 'plugin', ti.plugin, + 'arguments', ti.arguments + ) else null end, 'attachments', coalesce( (select json_group_array(json_object( 'id', a.id, @@ -1276,6 +1335,7 @@ def merged_log_rows( ) )) from tool_results tr + left join tool_instances ti on tr.instance_id = ti.id where tr.response_id = responses.id ), '[]' diff --git a/llm/migrations.py b/llm/migrations.py index f2c5f8837..a3262549b 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -834,3 +834,23 @@ def m027_parts_text_column(db): row["id"], {"text": text, "payload": json.dumps(payload) if payload else None}, ) + + +@migration +def m028_tool_instantiations(db): + # Which configured toolbox instance served a tool call - e.g. that + # SQLite_query ran against SQLite("mydb.db"). Local execution + # provenance, so it lives outside the hashed message tree, joined + # to the chain by tool_call_id (unique per call, stored on both the + # call and result parts). Deliberately the seed of a fuller + # execution-events table: duration or exception details would be + # additive columns here. + db["tool_instantiations"].create( + { + "tool_call_id": str, + "name": str, + "plugin": str, + "arguments": str, + }, + pk="tool_call_id", + ) diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 2afefb25e..87f518456 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -773,6 +773,34 @@ def test_messages_plus_prompt_both_reach_the_chain(self, store, mock_model): assert texts == ["original", "follow-up", "Hello"] assert store.verify() == [] + def test_log_to_db_records_tool_instantiations(self, store, mock_model): + class Notes(llm.Toolbox): + def __init__(self, path: str): + self.path = path + + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "next", + tool_results=[ + llm.ToolResult( + name="Notes_read", + output="hello", + tool_call_id="tc_1", + instance=Notes("/tmp/notes"), + ) + ], + ) + response.text() + response.log_to_db(store.db) + assert list(store.db["tool_instantiations"].rows) == [ + { + "tool_call_id": "tc_1", + "name": "Notes", + "plugin": None, + "arguments": '{"path": "/tmp/notes"}', + } + ] + def test_successive_library_turns_extend_the_thread(self, store, mock_model): conversation = mock_model.conversation() for reply in ("One", "Two"): diff --git a/tests/test_plugins.py b/tests/test_plugins.py index f82791dcf..671c802fc 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -475,12 +475,12 @@ def register_tools(self, register): ('{"tool_calls": [{"name": "upper", "arguments": {"text": "one"}}]}', "[]"), ( "", - '[{"id": ID, "tool_id": 1, "name": "upper", "output": "ONE", "tool_call_id": "tc_TCID", "exception": null, "attachments": []}]', + '[{"id": ID, "tool_id": 1, "name": "upper", "output": "ONE", "tool_call_id": "tc_TCID", "exception": null, "instance": null, "attachments": []}]', ), ('{"tool_calls": [{"name": "upper", "arguments": {"text": "two"}}]}', "[]"), ( "", - '[{"id": ID, "tool_id": 1, "name": "upper", "output": "TWO", "tool_call_id": "tc_TCID", "exception": null, "attachments": []}]', + '[{"id": ID, "tool_id": 1, "name": "upper", "output": "TWO", "tool_call_id": "tc_TCID", "exception": null, "instance": null, "attachments": []}]', ), ( '{"tool_calls": [{"name": "upper", "arguments": {"text": "three"}}]}', @@ -488,7 +488,7 @@ def register_tools(self, register): ), ( "", - '[{"id": ID, "tool_id": 1, "name": "upper", "output": "THREE", "tool_call_id": "tc_TCID", "exception": null, "attachments": []}]', + '[{"id": ID, "tool_id": 1, "name": "upper", "output": "THREE", "tool_call_id": "tc_TCID", "exception": null, "instance": null, "attachments": []}]', ), ) # Test the --td option @@ -818,8 +818,24 @@ def after_call(tool, tool_call, tool_result): "model": "echo", "tool_calls": [], "tool_results": [ - {"name": "Memory_set", "output": "null"}, - {"name": "Memory_get", "output": "two"}, + { + "name": "Memory_set", + "output": "null", + "instance": { + "name": "Memory", + "plugin": "ToolboxPlugin", + "arguments": "{}", + }, + }, + { + "name": "Memory_get", + "output": "two", + "instance": { + "name": "Memory", + "plugin": "ToolboxPlugin", + "arguments": "{}", + }, + }, ], }, { @@ -834,6 +850,11 @@ def after_call(tool, tool_call, tool_result): { "name": "Filesystem_list_files", "output": json.dumps([str(other_path)]), + "instance": { + "name": "Filesystem", + "plugin": "ToolboxPlugin", + "arguments": json.dumps({"path": str(my_dir2)}), + }, } ], }, @@ -925,9 +946,33 @@ def test_toolbox_logging_async(logs_db, tmpdir): "model": "echo", "tool_calls": [], "tool_results": [ - {"name": "Memory_set", "output": "null"}, - {"name": "Memory_get", "output": "two"}, - {"name": "Filesystem_list_files", "output": "[]"}, + { + "name": "Memory_set", + "output": "null", + "instance": { + "name": "Memory", + "plugin": "ToolboxPlugin", + "arguments": "{}", + }, + }, + { + "name": "Memory_get", + "output": "two", + "instance": { + "name": "Memory", + "plugin": "ToolboxPlugin", + "arguments": "{}", + }, + }, + { + "name": "Filesystem_list_files", + "output": "[]", + "instance": { + "name": "Filesystem", + "plugin": "ToolboxPlugin", + "arguments": json.dumps({"path": str(path)}), + }, + }, ], }, ] @@ -960,12 +1005,7 @@ def test_plugins_command(): def tool_activity_rows(db): """Per-turn tool calls and results from the message store, in the - shape the old TOOL_RESULTS_SQL produced from the legacy tables. - - Toolbox instance provenance is absent: the store does not record - which instance served a call - tool execution provenance is still - an open design decision. - """ + shape the old TOOL_RESULTS_SQL produced from the legacy tables.""" from llm.logs import LogStore, log_row_extras, merged_log_rows store = LogStore(db) @@ -982,7 +1022,11 @@ def tool_activity_rows(db): for call in extras["tool_calls"] ], "tool_results": [ - {"name": result["name"], "output": result["output"]} + { + "name": result["name"], + "output": result["output"], + "instance": result["instance"], + } for result in extras["tool_results"] ], } From 84c2837079cffcdccdb9517b776c8493ea4196ff Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 21:58:55 -0700 Subject: [PATCH 148/258] Chain round separator belongs to the chain, not to plugins Since May 2025 llm-anthropic yielded a literal space StreamEvent after a tool-calling round ("Stick a space in when tools run", refs #1019) so streamed chain output would not run one response's text into the next: "...can have dragons.Now that I...". A display concern expressed as a data event - under the message store that fake space became a real whitespace-only part row and participated in the message hash. ChainResponse and AsyncChainResponse now yield a single space at each round boundary where neither side brings its own whitespace, in both the text-chunk and stream-event iterators. Synthesized at the chain level, the separator never enters any response's recorded events, so nothing is stored or hashed - and async chains, which never had the plugin workaround, get the fix too. The corresponding llm-anthropic change (deleting the space yield and its snapshot artifact) is prepared in that repo, uncommitted. Tests that split chained echo output on the old "}{" junction now split on "} {". Co-Authored-By: Claude Fable 5 --- docs/changelog.md | 1 + llm/models.py | 71 +++++++++++++++++++++++++++++++++++++++++++-- tests/test_chat.py | 2 +- tests/test_tools.py | 33 ++++++++++++++++++--- 4 files changed, 100 insertions(+), 7 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index d0d9599fa..8dfa3d6cf 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,7 @@ ## Unreleased +- Chain responses now yield a single space at the boundary between rounds when neither side supplies its own whitespace, so streamed output no longer runs the end of one response into the start of the next. The separator is synthesized at the chain level for display only - previously plugins worked around this by emitting a real space event, which was recorded as a whitespace-only part in the log. [#1562](https://github.com/simonw/llm/pull/1562) - `llm logs --json` now shows which configured toolbox instance served each tool result - the toolbox name, plugin and constructor arguments, so a `SQLite_query` result records that it ran against `SQLite("mydb.db")`. Recorded in the new `tool_instantiations` table for new logs, and read from the legacy `tool_instances` data for history logged by older versions of LLM, which recorded it without ever displaying it. [#1562](https://github.com/simonw/llm/pull/1562) - The `parts` table stores literal text in a dedicated `text` column - raw and unescaped, so prose reads as prose in SQL queries - instead of wrapping it in JSON. The `payload` column now holds only structure (fragment references, tool call fields, provider metadata), without the redundant `type` key, and is NULL when the text column carries the whole part. Storage encoding only: message hashes are computed over resolved content and are unchanged. Existing databases are rewritten by a migration. [#1562](https://github.com/simonw/llm/pull/1562) - `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. Results are most-relevant first, with prompt matches weighted well above response matches; `-l/--latest` switches to most-recent first. Searches also cover history recorded by older versions of LLM through its original `responses_fts` index. A longstanding bug where search results were ordered weakest match first is fixed. See {ref}`logging-search`. [#1562](https://github.com/simonw/llm/pull/1562) diff --git a/llm/models.py b/llm/models.py index e9c6ab27d..1278255e4 100644 --- a/llm/models.py +++ b/llm/models.py @@ -2813,13 +2813,50 @@ def responses(self) -> Iterator[Response]: break def __iter__(self) -> Iterator[str]: + # Rounds of a chain are separate model responses; joined with + # nothing between them the text of one runs straight into the + # next ("...have dragons.Now that I..."). Yield one space at + # each boundary where neither side brings its own whitespace. + # Display only: the separator never enters any response's + # recorded events, so it is not stored or hashed. + last_char = "" for response_item in self.responses(): - yield from response_item + first_chunk = True + for chunk in response_item: + if not chunk: + continue + if ( + first_chunk + and last_char + and not last_char.isspace() + and not chunk[0].isspace() + ): + yield " " + first_chunk = False + yield chunk + last_char = chunk[-1] def stream_events(self): "Yield StreamEvents from every response in the chain." + from .parts import StreamEvent + + # The same round-boundary separator as __iter__, synthesized at + # the chain level so it is never part of a response's events. + last_char = "" for response_item in self.responses(): - yield from response_item.stream_events() + first_text = True + for event in response_item.stream_events(): + if event.type == "text" and event.chunk: + if ( + first_text + and last_char + and not last_char.isspace() + and not event.chunk[0].isspace() + ): + yield StreamEvent(type="text", chunk=" ") + first_text = False + last_char = event.chunk[-1] + yield event def text(self) -> str: return "".join(self) @@ -2911,14 +2948,44 @@ async def responses(self) -> AsyncIterator[AsyncResponse]: break async def __aiter__(self) -> AsyncIterator[str]: + # Round-boundary separator - same reasoning as the sync chain. + last_char = "" async for response_item in self.responses(): + first_chunk = True async for chunk in response_item: + if not chunk: + continue + if ( + first_chunk + and last_char + and not last_char.isspace() + and not chunk[0].isspace() + ): + yield " " + first_chunk = False yield chunk + last_char = chunk[-1] async def astream_events(self): "Yield StreamEvents from every response in the chain." + from .parts import StreamEvent + + # Same round-boundary separator as __aiter__, synthesized at the + # chain level so it is never part of a response's events. + last_char = "" async for response_item in self.responses(): + first_text = True async for event in response_item.astream_events(): + if event.type == "text" and event.chunk: + if ( + first_text + and last_char + and not last_char.isspace() + and not event.chunk[0].isspace() + ): + yield StreamEvent(type="text", chunk=" ") + first_text = False + last_char = event.chunk[-1] yield event async def text(self) -> str: diff --git a/tests/test_chat.py b/tests/test_chat.py index 0fd5b06fc..120f757f7 100644 --- a/tests/test_chat.py +++ b/tests/test_chat.py @@ -315,7 +315,7 @@ def upper(text: str) -> str: ' "attachments": [],\n' ' "stream": true,\n' ' "previous": []\n' - "}{\n" + "} {\n" ' "prompt": "",\n' ' "system": "",\n' ' "attachments": [],\n' diff --git a/tests/test_tools.py b/tests/test_tools.py index e086bf0de..afae2cf1c 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -114,6 +114,31 @@ def can_have_dragons(population: int) -> bool: assert third.tool_calls() == [] +def test_chain_round_separator_is_display_only(): + """The space between chain rounds is synthesized at the chain level + for display - it must never become a stored whitespace part.""" + + def hello(): + return "world" + + model = llm.get_model("echo") + chain_response = model.chain( + json.dumps({"tool_calls": [{"name": "hello"}]}), tools=[hello] + ) + events = list(chain_response.stream_events()) + text = "".join(e.chunk for e in events if e.type == "text") + # The separator reached the streamed output... + assert "\n} {\n" in text + + db = sqlite_utils.Database(memory=True) + migrate(db) + chain_response.log_to_db(db) + # ...but no whitespace-only text part was stored. + for row in db["parts"].rows: + if row["type"] == "text" and row["text"] is not None: + assert row["text"].strip(), row + + def test_tool_use_async_tool_function(): async def hello(): return "world" @@ -123,8 +148,8 @@ async def hello(): json.dumps({"tool_calls": [{"name": "hello"}]}), tools=[hello] ) output = chain_response.text() - # That's two JSON objects separated by '\n}{\n' - bits = output.split("\n}{\n") + # Two JSON objects, separated by the chain's round boundary space + bits = output.split("\n} {\n") assert len(bits) == 2 objects = [json.loads(bits[0] + "}"), json.loads("{" + bits[1])] tool_call_id = objects[1]["tool_results"][0]["tool_call_id"] @@ -167,8 +192,8 @@ async def hello2(): tools=[hello, hello2], ) output = await chain_response.text() - # That's two JSON objects separated by '\n}{\n' - bits = output.split("\n}{\n") + # Two JSON objects, separated by the chain's round boundary space + bits = output.split("\n} {\n") assert len(bits) == 2 objects = [json.loads(bits[0] + "}"), json.loads("{" + bits[1])] ids = [r["tool_call_id"] for r in objects[1]["tool_results"]] From c84dc5351a1ed5429bca7cacfcf0280e522c4a7d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 22:04:23 -0700 Subject: [PATCH 149/258] Address documentation review comments on the PR - Unreleased changelog bullets reordered most interesting first, and the merged-read bullet no longer claims -q is unavailable - Search docs just say search covers old and new conversations, without index mechanics - The worked example shows fixed example ULIDs rather than $TURN_N_ID placeholders, and the accompanying bullet explains turns use ULIDs without the legacy id-space history - The b2: prefix explanation cut to one sentence - The parts bullet says when text is stored inline versus as text_ref, instead of "borrows no fragments" - llm logs Markdown output shows the toolbox instance on each tool result - `SQLite({"path": "mydb.db"})` - not just --json - The Python logging example uses gpt-5.5 - The -o logprobs 3 section is dropped from the OpenAI docs entirely - now an undocumented feature Co-Authored-By: Claude Fable 5 --- docs/changelog.md | 12 ++++++------ docs/logging.md | 33 +++++++++++++++++---------------- docs/openai-models.md | 2 -- llm/cli.py | 11 ++++++++++- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 8dfa3d6cf..70966d4e3 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,15 +2,15 @@ ## Unreleased -- Chain responses now yield a single space at the boundary between rounds when neither side supplies its own whitespace, so streamed output no longer runs the end of one response into the start of the next. The separator is synthesized at the chain level for display only - previously plugins worked around this by emitting a real space event, which was recorded as a whitespace-only part in the log. [#1562](https://github.com/simonw/llm/pull/1562) -- `llm logs --json` now shows which configured toolbox instance served each tool result - the toolbox name, plugin and constructor arguments, so a `SQLite_query` result records that it ran against `SQLite("mydb.db")`. Recorded in the new `tool_instantiations` table for new logs, and read from the legacy `tool_instances` data for history logged by older versions of LLM, which recorded it without ever displaying it. [#1562](https://github.com/simonw/llm/pull/1562) -- The `parts` table stores literal text in a dedicated `text` column - raw and unescaped, so prose reads as prose in SQL queries - instead of wrapping it in JSON. The `payload` column now holds only structure (fragment references, tool call fields, provider metadata), without the redundant `type` key, and is NULL when the text column carries the whole part. Storage encoding only: message hashes are computed over resolved content and are unchanged. Existing databases are rewritten by a migration. [#1562](https://github.com/simonw/llm/pull/1562) -- `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. Results are most-relevant first, with prompt matches weighted well above response matches; `-l/--latest` switches to most-recent first. Searches also cover history recorded by older versions of LLM through its original `responses_fts` index. A longstanding bug where search results were ordered weakest match first is fixed. See {ref}`logging-search`. [#1562](https://github.com/simonw/llm/pull/1562) -- `llm logs` now reads both generations of log tables: history recorded by older versions of llm in the legacy `responses` table is merged into the output alongside new turns, with rows from the dual-write era deduplicated by id. The `-m`, `-c`, `-f`, `-T`, `--tools` and `--schema` filters work across both. `-q/--query` full-text search is temporarily unavailable while search against the new tables is designed. [#1562](https://github.com/simonw/llm/pull/1562) +- `llm logs` now reads both generations of log tables: history recorded by older versions of LLM in the legacy `responses` table is merged into the output alongside new turns, with rows from the dual-write era deduplicated by id. The `-m`, `-c`, `-f`, `-T`, `--tools` and `--schema` filters work across both. [#1562](https://github.com/simonw/llm/pull/1562) - Logging no longer writes the legacy tables: `Response.log_to_db()` records only the content-addressed message store. The legacy tables are kept in place as read-only history. Raw provider payloads (the old `prompt_json` and `response_json` columns) are no longer persisted - the stored message chain is the record of what was sent and returned. This means data that only ever lived in those raw payloads is no longer stored, notably the log probabilities returned by OpenAI completion models with `-o logprobs` - those remain available on the response object through the Python API, and rows logged by older versions of LLM keep the values they recorded. [#1562](https://github.com/simonw/llm/pull/1562) -- A response logged outside of a conversation now gets a thread of its own, so responses logged through the Python API can be continued with `llm -c` - the same guarantee the `conversations` table used to provide. [#1562](https://github.com/simonw/llm/pull/1562) +- `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. Results are most-relevant first, with prompt matches weighted well above response matches; `-l/--latest` switches to most-recent first. Searches also cover history recorded by older versions of LLM through its original `responses_fts` index. A longstanding bug where search results were ordered weakest match first is fixed. See {ref}`logging-search`. [#1562](https://github.com/simonw/llm/pull/1562) +- The `parts` table stores literal text in a dedicated `text` column - raw and unescaped, so prose reads as prose in SQL queries - instead of wrapping it in JSON. The `payload` column now holds only structure (fragment references, tool call fields, provider metadata), without the redundant `type` key, and is NULL when the text column carries the whole part. Storage encoding only: message hashes are computed over resolved content and are unchanged. Existing databases are rewritten by a migration. [#1562](https://github.com/simonw/llm/pull/1562) +- `llm logs` now shows which configured toolbox instance served each tool result - the toolbox name, plugin and constructor arguments, so a `SQLite_query` result records that it ran against `SQLite("mydb.db")`. Recorded in the new `tool_instantiations` table for new logs, and read from the legacy `tool_instances` data for history logged by older versions of LLM, which recorded it without ever displaying it. [#1562](https://github.com/simonw/llm/pull/1562) - New documentation for the message store: the schema, the content-addressing hash contract, worked examples and a SQL cookbook, in {ref}`the logging documentation `. [#1562](https://github.com/simonw/llm/pull/1562) - Passing `prompt=`, `fragments=`, `attachments=` or `tool_results=` alongside `messages=` to `model.prompt()` or `conversation.prompt()` now appends that new input to the supplied message history, instead of silently omitting it from `prompt.messages` - previously the model could receive text that never appeared in the logged conversation. [#1562](https://github.com/simonw/llm/pull/1562) +- A response logged outside of a conversation now gets a thread of its own, so responses logged through the Python API can be continued with `llm -c` - the same guarantee the `conversations` table used to provide. [#1562](https://github.com/simonw/llm/pull/1562) +- Chain responses now yield a single space at the boundary between rounds when neither side supplies its own whitespace, so streamed output no longer runs the end of one response into the start of the next. The separator is synthesized at the chain level for display only - previously plugins worked around this by emitting a real space event, which was recorded as a whitespace-only part in the log. [#1562](https://github.com/simonw/llm/pull/1562) (v0_31_1)= ## 0.31.1 (2026-07-09) diff --git a/docs/logging.md b/docs/logging.md index 95fa4b712..eb8e6c18f 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -191,7 +191,7 @@ To switch to sorting with most recent first, add `-l/--latest`. This can be comb llm logs -q 'cheesecake' -l -n 3 ``` -Search spans both generations of log tables: new conversations are matched through the `turn_search` index over the content-addressed tables, and history recorded by older versions of LLM is matched through its original `responses_fts` index, with the two result sets merged. The relevance scores come from two separate indexes, so the interleaving between very old and new results is approximate. +Search covers both new conversations and history recorded by older versions of LLM. (logging-filter-id)= @@ -335,13 +335,14 @@ for text in REPLIES: response.text() response.log_to_db(db) -# ULID identifiers and timestamps differ on every run, so they are -# replaced with placeholders. The hashes are deterministic. +# ULID identifiers differ on every run, so they are replaced with +# fixed example ULIDs. The hashes are deterministic. +example_ulids = ["01kf2rw8jj3nfd5t7w9y1a3c5e", "01kf2rw8jkq7h9k2m4n6p8r0t2"] aliases = {} for i, row in enumerate(db.query("select id from turns order by id")): - aliases[row["id"]] = "$TURN_{}_ID".format(i + 1) + aliases[row["id"]] = example_ulids[i] thread = next(db.query("select * from threads")) -aliases[thread["id"]] = "$THREAD_ID" +aliases[thread["id"]] = "01kf2rw8jhv1x9c2m4p6q8s0tv" lines = ["messages and their parts:"] for row in db.query("select * from messages"): @@ -372,7 +373,7 @@ for row in db.query("select * from turns order by id"): lines.append("") lines.append("thread:") lines.append("") -lines.append("thread $THREAD_ID") +lines.append("thread {}".format(aliases[thread["id"]])) lines.append(" name: {}".format(thread["name"])) lines.append(" tip_message_hash: {}".format(thread["tip_message_hash"])) cog.out("```\n{}\n```\n".format("\n".join(lines))) @@ -398,21 +399,21 @@ assistant b2:a30a236e0d1b717c592e826d06e3c9d2 turns: -turn $TURN_1_ID - thread_id: $THREAD_ID +turn 01kf2rw8jj3nfd5t7w9y1a3c5e + thread_id: 01kf2rw8jhv1x9c2m4p6q8s0tv parent_message_hash: b2:d6b0cd4e7a65ea90423c50fadb3f5704 tip_message_hash: b2:0f2c02ad982050b623b7e034199c8c61 model: scripted -turn $TURN_2_ID - thread_id: $THREAD_ID +turn 01kf2rw8jkq7h9k2m4n6p8r0t2 + thread_id: 01kf2rw8jhv1x9c2m4p6q8s0tv parent_message_hash: b2:c785dd6c77540150c2647f406cacc76f tip_message_hash: b2:a30a236e0d1b717c592e826d06e3c9d2 model: scripted thread: -thread $THREAD_ID +thread 01kf2rw8jhv1x9c2m4p6q8s0tv name: Suggest a name for a pet pelican tip_message_hash: b2:a30a236e0d1b717c592e826d06e3c9d2 ``` @@ -424,7 +425,7 @@ Things to notice: - The `part N:` lines show each part's storage columns. A part whose text is pure literal keeps it in the `text` column - raw, unescaped and never parsed, so text that happens to look like JSON is safe - with a `null` payload. The `payload` column holds any remaining structure as JSON: fragment references, tool call fields, provider metadata. The part's type lives only in the `type` column. - Each turn brackets one model call. Its `parent_message_hash` is the tip of the chain that was sent to the model - the last input message, usually that turn's user prompt - and its `tip_message_hash` is the chain tip after the model's reply was appended. The prompt and response that `llm logs` displays are derived by splitting the chain at the parent, rather than being stored a second time. - The thread's id is the conversation id, its name is derived from the first prompt and its `tip_message_hash` follows the head of the conversation as new turns are logged. -- The ULID identifiers - sortable random identifiers issued in time order, the same id space older versions used for response ids - and the timestamp columns have been replaced with placeholders because they change on every run. The hashes have not: they depend only on the message content and its position in the chain, so replaying this conversation produces these exact four hashes. +- Turn and thread ids are ULIDs: sortable identifiers issued in time order. The ids shown here are illustrative, since fresh ones are generated on every run. The hashes are not - they depend only on the message content and its position in the chain, so replaying this conversation produces these exact four hashes. (logging-message-store-hashes)= @@ -436,7 +437,7 @@ A message's hash is calculated like this: 2. Serialize that object to canonical JSON: keys sorted, compact `,` and `:` separators with no extra whitespace, non-ASCII characters left unescaped. 3. Hash the UTF-8 encoding of that string with [BLAKE2b](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) using a 16 byte digest, and prefix the hex digest with `b2:`. -The `b2:` prefix tags the hash with the algorithm that produced it. If a future version of LLM ever changes the digest or the canonical form, the change will be detectable instead of silently splitting the stored data into two incompatible halves. +The `b2:` prefix names the algorithm that produced the hash, so any future change to it will be detectable. Two design decisions matter here: @@ -536,14 +537,14 @@ Attachments work the same way: the binary content lives in the `attachments` tab The full schema for these tables appears in {ref}`the SQL schema section ` below. - `messages` - one row per unique message. `hash` is the content address described above, `parent_hash` links to the previous message in the chain and `role` is `system`, `user` or `assistant`. `provider_metadata` holds any provider-specific data carried by the message; it participates in the hash. -- `parts` - the content of each message, ordered by `position`. `text` holds the part's literal text when it borrows no fragments - raw and never parsed, so `select text from parts` reads as prose. `payload` holds whatever structure remains as JSON (fragment references, tool call fields, provider metadata), in the order the model produced it, or NULL when the text column carries the whole part. `type` and `tool_name` are their own columns for direct filtering; the type never appears inside the payload. +- `parts` - the content of each message, ordered by `position`. When a part's text is stored inline in full it lives in the `text` column - raw and never parsed, so `select text from parts` reads as prose. Text that references fragments is stored in `payload` as `text_ref` instead. `payload` holds any remaining structure as JSON (fragment references, tool call fields, provider metadata), or NULL when the text column carries the whole part. `type` and `tool_name` are their own columns for direct filtering; the type never appears inside the payload. - `part_attachments` and `part_fragments` - junction tables recording which rows in `attachments` and `fragments` a part's payload references, in order. - `threads` - one row per conversation. `id` is the conversation id, `tip_message_hash` points at the current head of the conversation and `forked_from` records the thread a fork came from. - `turns` - one row per model call. `parent_message_hash` and `tip_message_hash` bracket the call's input and output as described above, and the remaining columns record provenance: model, options, schema, token counts and timings. Turn ids are ULIDs, in the same id space as legacy response ids, which is how the two generations of tables sort together in `llm logs`. - `turn_tools` - which {ref}`tool ` definitions were available to a turn, referencing the `tools` table. - `turn_fragments` - which fragments a turn was given, with their `kind` (`prompt` or `system`) and order. Provenance lives here rather than on the shared message rows, and this table is what powers `llm logs -f`. - `turn_search` - the searchable text of each turn: the literal prompt the user typed (fragment content excluded, via the `text_ref` literals described above) and the assistant's text output. An FTS5 index over this table, `turn_search_fts`, is what powers {ref}`llm logs -q `. Derived from the stored parts when a turn is logged; a turn with no prompt or response text, such as a pure tool call, gets no row. -- `tool_instantiations` - which configured {ref}`toolbox ` instance served a tool call: the toolbox name, its plugin and its constructor arguments, keyed by `tool_call_id`. Message rows are shared between conversations and so cannot carry this kind of local execution provenance; this table joins to the chain from outside it, and is what lets `llm logs --json` show that a `SQLite_query` call ran against `SQLite("mydb.db")`. +- `tool_instantiations` - which configured {ref}`toolbox ` instance served a tool call: the toolbox name, its plugin and its constructor arguments, keyed by `tool_call_id`. Message rows are shared between conversations and so cannot carry this kind of local execution provenance; this table joins to the chain from outside it, and is what lets `llm logs` show that a `SQLite_query` call ran against `SQLite("mydb.db")`. (logging-message-store-queries)= @@ -637,7 +638,7 @@ import llm import sqlite_utils db = sqlite_utils.Database("logs.db") -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.5") response = model.prompt("A short pelican fact") print(response.text()) response.log_to_db(db) diff --git a/docs/openai-models.md b/docs/openai-models.md index 33b269c52..7b4a8fa1a 100644 --- a/docs/openai-models.md +++ b/docs/openai-models.md @@ -123,8 +123,6 @@ The vector size of the supported OpenAI embedding models are as follows: The `gpt-3.5-turbo-instruct` model is a little different - it is a completion model rather than a chat model, described in [the OpenAI completions documentation](https://platform.openai.com/docs/api-reference/completions/create). -Completion models can be called with the `-o logprobs 3` option (not supported by chat models) which will request 3 log probabilities for each returned token. These are no longer stored in the SQLite log database - raw provider payloads are not persisted there - but they can be read at runtime from `response.response_json` when using the {ref}`Python API `. Rows logged by older versions of LLM retain the values they stored; consult [this issue](https://github.com/simonw/llm/issues/284#issuecomment-1724772704) for how to read those. - (openai-extra-models)= ## Adding more OpenAI models diff --git a/llm/cli.py b/llm/cli.py index 6476b3464..52599352d 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -2145,10 +2145,19 @@ def _display_fragments(fragments, title): elif attachment.get("content"): desc += f"<{attachment['content_length']:,} bytes>" attachments += f"\n - {desc}" + instance = tool_result.get("instance") + instance_bit = "" + if instance: + arguments = instance["arguments"] + instance_bit = " - instance `{}({})`".format( + instance["name"], + arguments if arguments and arguments != "{}" else "", + ) click.echo( - "- **{}**: `{}`
\n{}{}{}".format( + "- **{}**: `{}`{}
\n{}{}{}".format( tool_result["name"], tool_result["tool_call_id"], + instance_bit, _fenced_block(tool_result["output"]), ( "
\n **Error**: {}\n".format( From 3cfba782b309535268eaeb3eef6a8a414162b8e9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 22:18:58 -0700 Subject: [PATCH 150/258] Hash attachments by content, not filesystem path message_hash serialized a path-backed attachment as its path, so editing the file afterwards left the stored hash looking valid while the attachment id pointed at the old bytes - verify() could not see it, and the same bytes at two paths were two identities while different bytes at one path were one. The canonical form hashed now represents every attachment as {"id": ...}: the sha256 of its bytes, the same id that keys the attachments table. Wire serialization (to_dict/from_dict) is unchanged - only the hashed form differs. An attachment supplied as a URL hashes the URL itself, a deliberate and now documented decision: the log records which URL was sent, not whatever it served that day. Hashes of messages without attachments are unchanged, including every hash in the documentation's worked examples. For databases written by earlier development versions of this branch, m029 recomputes every stored hash from resolved content bottom-up, repoints parts, turns, threads and parent links, and merges messages that only ever differed by attachment path. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- docs/logging.md | 2 +- llm/logs.py | 25 +++++++- llm/migrations.py | 105 +++++++++++++++++++++++++++++++ tests/test_logs_store.py | 130 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 260 insertions(+), 2 deletions(-) diff --git a/docs/logging.md b/docs/logging.md index eb8e6c18f..808449821 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -441,7 +441,7 @@ The `b2:` prefix names the algorithm that produced the hash, so any future chang Two design decisions matter here: -- **The hash covers resolved content.** {ref}`Fragment ` references and attachment references are expanded before hashing, so the hash always covers exactly the content the model saw, never the compressed form it happens to be stored in. The internal `LogStore.verify()` method re-derives every hash from the stored rows to check this stays true. +- **The hash covers resolved content.** {ref}`Fragment ` references are expanded to their full text before hashing, and attachments are represented by the SHA-256 hash of their bytes - never by the filesystem path they were loaded from, so editing a file after logging cannot leave a stale hash looking valid. An attachment supplied as a URL is hashed by that URL: the log records which URL was sent, not whatever it served that day. The internal `LogStore.verify()` method re-derives every hash from the stored rows to check all of this stays true. - **The parent hash participates in the hash.** The same content appearing at a different point in a conversation is a different node. This is what makes two conversations that share a prefix collapse to shared rows, with no explicit comparison required: replaying the same messages produces the same hashes, and a hash that is already present needs nothing written. A consequence of the second decision is that a stateless client - one that holds its own conversation history and re-sends the whole thing with every call - writes only the new tail on each request. It also makes forks cheap, as described next. Here the first conversation from the worked example is logged again, followed by a second conversation that starts with the same prompt and then diverges: diff --git a/llm/logs.py b/llm/logs.py index 95539e6e3..76f29bea8 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -74,14 +74,37 @@ def content_hash(obj: Any) -> str: return f"{HASH_PREFIX}{digest}" +def _canonical_attachment(attachment) -> dict: + """The hashed form of an attachment: its content id. + + Identity is the sha256 of the bytes - the same id that keys the + attachments table - never the filesystem path they happened to live + at, so editing a file after logging cannot leave a stale hash + looking valid, and the same bytes at two paths are one identity. + URL attachments hash the URL itself: the log records which URL was + sent, not whatever that URL served on the day. + """ + return {"id": attachment.id()} + + def message_hash(message: Message, parent_hash: str | None) -> str: """Identity of ``message`` when reached via ``parent_hash``. The parent participates, so the same content at a different point in a conversation is a different node. That is what makes a shared prefix collapse to shared rows without any explicit comparison. + + Attachments are hashed by content id, via _canonical_attachment. """ - return content_hash({"parent": parent_hash, "message": message.to_dict()}) + d: Any = message.to_dict() + for part, part_dict in zip(message.parts, d["parts"]): + attachment = getattr(part, "attachment", None) + if attachment is not None: + part_dict["attachment"] = _canonical_attachment(attachment) + attachments = getattr(part, "attachments", None) + if attachments: + part_dict["attachments"] = [_canonical_attachment(a) for a in attachments] + return content_hash({"parent": parent_hash, "message": d}) class LogStore: diff --git a/llm/migrations.py b/llm/migrations.py index a3262549b..f2886caa2 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -854,3 +854,108 @@ def m028_tool_instantiations(db): }, pk="tool_call_id", ) + + +@migration +def m029_rehash_messages(db): + # Message hashes now identify attachments by the sha256 of their + # content rather than the filesystem path they were loaded from. + # Recompute every stored hash from resolved content, bottom-up, and + # repoint everything that references one. Hashing from content also + # merges messages that only ever differed by attachment path. + if not db["messages"].exists() or not db["messages"].count: + return + # Runtime import - llm.logs imports this module at import time, but + # by the time a migration runs both modules are fully loaded. + from .logs import LogStore, message_hash + from .parts import Message + + store = LogStore.__new__(LogStore) # skip __init__, which migrates + store.db = db + + rows = {row["hash"]: row for row in db["messages"].rows} + parts_by_hash = store._load_parts(list(rows)) + + children: dict = {} + roots = [] + for row in rows.values(): + if row["parent_hash"] is None: + roots.append(row["hash"]) + else: + children.setdefault(row["parent_hash"], []).append(row["hash"]) + + mapping: dict = {} + queue = list(roots) + while queue: + old_hash = queue.pop() + row = rows[old_hash] + parent = row["parent_hash"] + message = Message( + role=row["role"], + parts=parts_by_hash.get(old_hash, []), + provider_metadata=_load_json(row["provider_metadata"]), + ) + mapping[old_hash] = message_hash(message, mapping.get(parent, parent)) + queue.extend(children.get(old_hash, [])) + + with db.conn: + seen: set = set() + for old_hash, new_hash in mapping.items(): + if new_hash in seen: + # Two messages that differed only by attachment path + # are now one identity - keep the first, drop this + # one's rows and repoint its references below. + part_ids = [ + r["id"] + for r in db.query( + "select id from parts where message_hash = ?", [old_hash] + ) + ] + if part_ids: + placeholders = ",".join("?" * len(part_ids)) + db.execute( + f"delete from part_attachments where part_id in ({placeholders})", + part_ids, + ) + db.execute( + f"delete from part_fragments where part_id in ({placeholders})", + part_ids, + ) + db.execute( + f"delete from parts where id in ({placeholders})", part_ids + ) + db.execute("delete from messages where hash = ?", [old_hash]) + continue + seen.add(new_hash) + if new_hash != old_hash: + db.execute( + "update messages set hash = ? where hash = ?", [new_hash, old_hash] + ) + db.execute( + "update parts set message_hash = ? where message_hash = ?", + [new_hash, old_hash], + ) + for old_hash, new_hash in mapping.items(): + if new_hash == old_hash: + continue + db.execute( + "update messages set parent_hash = ? where parent_hash = ?", + [new_hash, old_hash], + ) + db.execute( + "update turns set parent_message_hash = ? " + "where parent_message_hash = ?", + [new_hash, old_hash], + ) + db.execute( + "update turns set tip_message_hash = ? where tip_message_hash = ?", + [new_hash, old_hash], + ) + db.execute( + "update threads set tip_message_hash = ? where tip_message_hash = ?", + [new_hash, old_hash], + ) + + +def _load_json(value): + return json.loads(value) if value else None diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 87f518456..953d94890 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -814,6 +814,136 @@ def test_successive_library_turns_extend_the_thread(self, store, mock_model): # ---- storage by reference -------------------------------------------- +class TestAttachmentHashing: + """Message identity covers attachment content, never the filesystem + path the bytes were loaded from.""" + + def test_same_bytes_at_two_paths_are_one_identity(self, tmp_path): + path_a = tmp_path / "a.png" + path_b = tmp_path / "b.png" + path_a.write_bytes(b"SAME BYTES") + path_b.write_bytes(b"SAME BYTES") + hashes = [ + message_hash( + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type="image/png", path=str(p)) + ) + ], + ), + None, + ) + for p in (path_a, path_b) + ] + assert hashes[0] == hashes[1] + + def test_changed_bytes_at_the_same_path_change_the_hash(self, tmp_path): + path = tmp_path / "x.png" + + def hash_now(): + return message_hash( + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type="image/png", path=str(path)) + ) + ], + ), + None, + ) + + path.write_bytes(b"first version") + first = hash_now() + path.write_bytes(b"second version") + assert hash_now() != first + + def test_same_bytes_at_two_paths_share_the_stored_row(self, store, tmp_path): + for name in ("a.png", "b.png"): + path = tmp_path / name + path.write_bytes(b"SAME BYTES") + store.ensure_chain( + [ + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type="image/png", path=str(path)) + ) + ], + ) + ] + ) + assert store.db["messages"].count == 1 + + def test_attachment_chain_verifies(self, store, tmp_path): + path = tmp_path / "x.png" + path.write_bytes(b"PNG BYTES") + store.ensure_chain( + [ + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type="image/png", path=str(path)) + ) + ], + ), + llm.assistant("A fine image"), + ] + ) + assert store.verify() == [] + + def test_m029_recomputes_stale_hashes(self, store, tmp_path): + # Build a real chain, then rewrite its hashes to bogus values - + # simulating rows written by the old path-based algorithm - and + # check the migration recomputes everything from content. + path = tmp_path / "x.png" + path.write_bytes(b"PNG BYTES") + messages = [ + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type="image/png", path=str(path)) + ) + ], + ), + llm.assistant("A fine image"), + ] + tip = store.ensure_chain(messages) + thread_id = store.create_thread(name="t", tip=tip) + db = store.db + real = [row["hash"] for row in db.query("select hash from messages")] + fakes = {h: "b2:" + format(i, "032x") for i, h in enumerate(real)} + with db.conn: + for old, fake in fakes.items(): + db.execute("update messages set hash = ? where hash = ?", [fake, old]) + db.execute( + "update messages set parent_hash = ? where parent_hash = ?", + [fake, old], + ) + db.execute( + "update parts set message_hash = ? where message_hash = ?", + [fake, old], + ) + db.execute( + "update threads set tip_message_hash = ? " + "where tip_message_hash = ?", + [fake, old], + ) + db.execute( + "delete from _llm_migrations where name = 'm029_rehash_messages'" + ) + assert store.verify() != [] + migrate(db) + assert store.verify() == [] + assert store.thread_tip(thread_id) == tip + assert store.load_chain(tip) == messages + + class TestPartStorageFormat: """Literal text is stored raw in its own column - never escaped, never parsed - and the JSON payload holds only structure, with the From a6394aed98769dd214ba5b8dd64b57b56e584cb9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 22:23:46 -0700 Subject: [PATCH 151/258] Make the m027 parts migration safe to retry Each row update committed independently, so an interrupted run that retried would find already-migrated rows - whose payloads no longer carry a text key - and blank their text column back to None. The migration now runs in one transaction and skips any row that no longer carries the old keys, making a retry a no-op for migrated rows. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- llm/migrations.py | 25 +++++++++++++++---------- tests/test_logs_store.py | 21 +++++++++++++++++++++ 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/llm/migrations.py b/llm/migrations.py index f2886caa2..073eb5cfc 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -824,16 +824,21 @@ def m027_parts_text_column(db): # message content before anything is written, so no hash changes. if "text" not in db["parts"].columns_dict: db["parts"].add_column("text", str) - for row in list(db.query("select id, type, payload from parts")): - payload = json.loads(row["payload"]) if row["payload"] else {} - payload.pop("type", None) - text = None - if row["type"] in ("text", "reasoning") and "text" in payload: - text = payload.pop("text") - db["parts"].update( - row["id"], - {"text": text, "payload": json.dumps(payload) if payload else None}, - ) + # One transaction, and each row is only touched if it still carries + # the old keys - so an interrupted run can be retried without the + # already-migrated rows (whose payloads no longer have a text key) + # being blanked back to text=None. + with db.conn: + for row in list(db.query("select id, type, payload from parts")): + payload = json.loads(row["payload"]) if row["payload"] else {} + if "type" not in payload and "text" not in payload: + continue + payload.pop("type", None) + update: dict = {} + if row["type"] in ("text", "reasoning") and "text" in payload: + update["text"] = payload.pop("text") + update["payload"] = json.dumps(payload) if payload else None + db["parts"].update(row["id"], update) @migration diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 953d94890..e74f79a58 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -814,6 +814,27 @@ def test_successive_library_turns_extend_the_thread(self, store, mock_model): # ---- storage by reference -------------------------------------------- +class TestMigrationRetrySafety: + def test_m027_retry_does_not_erase_migrated_text(self, store, mock_model): + # Simulate an interrupted m027: rows already in the new format + # but the migration not recorded as applied. Retrying must not + # blank the text column back out. + mock_model.enqueue(["Hello there"]) + response = mock_model.prompt("Hi") + response.text() + response.log_to_db(store.db) + before = {row["id"]: row["text"] for row in store.db["parts"].rows} + assert any(before.values()) + with store.db.conn: + store.db.execute( + "delete from _llm_migrations where name = 'm027_parts_text_column'" + ) + migrate(store.db) + after = {row["id"]: row["text"] for row in store.db["parts"].rows} + assert after == before + assert store.verify() == [] + + class TestAttachmentHashing: """Message identity covers attachment content, never the filesystem path the bytes were loaded from.""" From bcd5dd8d07ad4c1d3a914f5fddce3a962fe83853 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 22:26:39 -0700 Subject: [PATCH 152/258] Completion models build their prompt from prompt.messages Completion.execute concatenated history by iterating conversation.responses, which is empty for a conversation reloaded from the message store - so llm -c with a completion model sent only the newest question. It now reads prompt.messages, which carries the full chain under the prompt.messages invariant, reloaded history included. Chat and Responses models already worked this way. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- llm/default_plugins/openai_models.py | 19 ++++++++++++++----- tests/conftest.py | 1 + tests/test_llm.py | 18 ++++++++++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 30583150f..635025761 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1897,12 +1897,21 @@ def execute( raise NotImplementedError( "System prompts are not supported for OpenAI completion models" ) + from llm.parts import TextPart + + # prompt.messages carries the full history - including history + # reloaded from storage, which conversation.responses does not. messages = [] - if conversation is not None: - for prev_response in conversation.responses: - messages.append(prev_response.prompt.prompt) - messages.append(cast(Response, prev_response).text()) - messages.append(prompt.prompt) + for message in prompt.messages: + if message.role not in ("user", "assistant"): + continue + text = "".join( + part.text + for part in message.parts + if isinstance(part, TextPart) and part.text + ) + if text: + messages.append(text) kwargs = self.build_kwargs(prompt, stream) client = self.get_client(key) if stream: diff --git a/tests/conftest.py b/tests/conftest.py index a58240887..1b7af85dc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -337,6 +337,7 @@ def mocked_openai_completion(httpx_mock): "usage": {"prompt_tokens": 5, "completion_tokens": 7, "total_tokens": 12}, }, headers={"Content-Type": "application/json"}, + is_reusable=True, ) return httpx_mock diff --git a/tests/test_llm.py b/tests/test_llm.py index aba3c3319..633fc1a4c 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -276,6 +276,24 @@ def test_openai_completion(mocked_openai_completion, user_path): ) +def test_openai_completion_continue_includes_history( + mocked_openai_completion, user_path +): + # A continued conversation reloaded from storage must send the + # prior exchanges, not just the newest prompt - prompt.messages + # carries them; conversation.responses does not. + runner = CliRunner() + base = ["-m", "gpt-3.5-turbo-instruct", "--no-stream", "--key", "x"] + result = runner.invoke(cli, base + ["Say this is a test"], catch_exceptions=False) + assert result.exit_code == 0 + result2 = runner.invoke(cli, base + ["Say it again", "-c"], catch_exceptions=False) + assert result2.exit_code == 0 + body = json.loads(mocked_openai_completion.get_requests()[-1].content) + assert body["prompt"] == ( + "Say this is a test\n\n\nThis is indeed a test\nSay it again" + ) + + def test_openai_completion_system_prompt_error(): runner = CliRunner() result = runner.invoke( From b60b80d8cb94bc36f955271dfc5b97f000b7f41c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 22:29:13 -0700 Subject: [PATCH 153/258] A turn owns the tool results at the head of its input segment A turn's input can end [tool results, user prompt] - the reply() and prompt-fold shapes - but display derived the turn's input from the last input message alone, and the -T filter checked only the parent message's parts. The tool results one message up were invisible: not shown by llm logs, not matched by -T or --tools. The reader now derives the turn's own input as the trailing run of user and tool messages - everything after the last assistant or system message belongs to the turn, because a turn's new input never contains an assistant message. Display prompt text comes from the user messages of that segment, extras from all of it, and the -T clause checks the parent's own parent when the parent is a user message. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- llm/logs.py | 32 ++++++++++++++++++++++++++++---- tests/test_logs_store.py | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/llm/logs.py b/llm/logs.py index 76f29bea8..652481574 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -789,7 +789,23 @@ def build(self, row: dict) -> dict: inputs = self.store.load_chain(row["parent_message_hash"]) outputs = self.store.load_chain(row["tip_message_hash"])[len(inputs) :] - prompt_parts = inputs[-1].parts if inputs else [] + # The turn's own input is the trailing run of user and tool + # messages: everything after the last assistant (or system) + # message belongs to this turn, because a turn's new input + # never contains an assistant message. A turn carrying both + # tool results and a fresh user prompt therefore keeps both. + boundary = len(inputs) + while boundary and inputs[boundary - 1].role in ("user", "tool"): + boundary -= 1 + input_messages = inputs[boundary:] + + prompt_parts = [ + part + for message in input_messages + if message.role == "user" + for part in message.parts + ] + input_parts = [part for message in input_messages for part in message.parts] system_parts = inputs[0].parts if inputs and inputs[0].role == "system" else [] out_parts = [part for message in outputs for part in message.parts] @@ -829,7 +845,7 @@ def build(self, row: dict) -> dict: # redundant with it. "prompt_json": None, "response_json": None, - "_input_parts": prompt_parts, + "_input_parts": input_parts, "_output_parts": out_parts, # Internal, stripped before rendering - the enrichment # needs them to find the parts rows behind these parts. @@ -935,10 +951,18 @@ def log_rows( def _tool_result_clause(extra: str = "") -> str: - return f"""turns.parent_message_hash in ( + # A turn's tool results sit either in its parent message directly, + # or - when a fresh user prompt followed the results - in the + # parent's own parent, one step further up the same input segment. + return f"""(turns.parent_message_hash in ( select parts.message_hash from parts where parts.type = 'tool_result' {extra} - )""" + ) or turns.parent_message_hash in ( + select messages.hash from messages + join parts on parts.message_hash = messages.parent_hash + where messages.role = 'user' + and parts.type = 'tool_result' {extra} + ))""" def log_row_extras(store: "LogStore", row: dict) -> dict: diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index e74f79a58..6f251fd2c 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -14,7 +14,13 @@ import llm from llm.cli import cli -from llm.logs import LogStore, canonical_json, message_hash +from llm.logs import ( + LogStore, + canonical_json, + log_row_extras, + merged_log_rows, + message_hash, +) from llm.migrations import migrate from llm.models import Attachment from llm.parts import ( @@ -814,6 +820,35 @@ def test_successive_library_turns_extend_the_thread(self, store, mock_model): # ---- storage by reference -------------------------------------------- +class TestTurnInputBoundary: + """A turn whose input ends [tool results, user prompt] owns both - + the tool results must not vanish from display or the -T filter just + because a user message follows them.""" + + def _log_turn_with_results_and_prompt(self, store, mock_model): + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "next question", + messages=[llm.user("orig"), llm.assistant("first answer")], + tool_results=[llm.ToolResult(name="t", output="RESULT", tool_call_id="c9")], + ) + response.text() + response.log_to_db(store.db) + + def test_tool_results_and_prompt_both_display(self, store, mock_model): + self._log_turn_with_results_and_prompt(store, mock_model) + row = merged_log_rows(store)[0] + assert row["prompt"] == "next question" + extras = log_row_extras(store, row) + assert [result["output"] for result in extras["tool_results"]] == ["RESULT"] + + def test_tool_filters_match(self, store, mock_model): + self._log_turn_with_results_and_prompt(store, mock_model) + assert len(merged_log_rows(store, any_tools=True)) == 1 + assert len(merged_log_rows(store, tool_names=["t"])) == 1 + assert merged_log_rows(store, tool_names=["other"]) == [] + + class TestMigrationRetrySafety: def test_m027_retry_does_not_erase_migrated_text(self, store, mock_model): # Simulate an interrupted m027: rows already in the new format From dee097016885a7e720e92a0f18de530278fa3b82 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 22:31:32 -0700 Subject: [PATCH 154/258] Scope tool_instantiations by turn, not tool_call_id alone The synthesized tc_ ids are unique, but provider-supplied ids pass through unchanged, and providers with per-request counters can reuse the same id across independent turns - a later call would overwrite an earlier call's recorded toolbox provenance. The table is now keyed by (turn_id, tool_call_id), writes carry the turn and reads are scoped to it. m030 backfills turn ids for existing rows by matching the stored parts, dropping any row it cannot place. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- llm/logs.py | 19 ++++++++++++++----- llm/migrations.py | 27 +++++++++++++++++++++++++++ tests/test_logs_store.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/llm/logs.py b/llm/logs.py index 652481574..a9abeb85a 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -512,6 +512,7 @@ def log(self, response, thread_id: str | None = None) -> str: continue self.db["tool_instantiations"].insert( { + "turn_id": turn_id, "tool_call_id": tool_result.tool_call_id, "name": tool_result.name.split("_")[0], "plugin": next( @@ -1000,7 +1001,9 @@ def log_row_extras(store: "LogStore", row: dict) -> dict: part for part in row.get("_input_parts", []) if isinstance(part, ToolResultPart) ] instances = _instances_by_tool_call_id( - store, [part.tool_call_id for part in result_parts if part.tool_call_id] + store, + row["id"], + [part.tool_call_id for part in result_parts if part.tool_call_id], ) tool_results = [ { @@ -1093,8 +1096,14 @@ def _tool_ids_by_name(store: "LogStore") -> dict: } -def _instances_by_tool_call_id(store: "LogStore", tool_call_ids: list) -> dict: - "Which configured toolbox instance served each call, for display." +def _instances_by_tool_call_id( + store: "LogStore", turn_id: str, tool_call_ids: list +) -> dict: + """Which configured toolbox instance served each call, for display. + + Scoped to the turn: providers with per-request counters can reuse + the same tool_call_id across independent turns. + """ if not tool_call_ids: return {} placeholders = ",".join("?" * len(tool_call_ids)) @@ -1108,9 +1117,9 @@ def _instances_by_tool_call_id(store: "LogStore", tool_call_ids: list) -> dict: f""" select tool_call_id, name, plugin, arguments from tool_instantiations - where tool_call_id in ({placeholders}) + where turn_id = ? and tool_call_id in ({placeholders}) """, - tool_call_ids, + [turn_id] + tool_call_ids, ) } diff --git a/llm/migrations.py b/llm/migrations.py index 073eb5cfc..25dd9d8d7 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -964,3 +964,30 @@ def m029_rehash_messages(db): def _load_json(value): return json.loads(value) if value else None + + +@migration +def m030_tool_instantiations_turn_scope(db): + # tool_call_id is not globally unique - providers with per-request + # counters can reuse the same id across independent turns - so the + # table is keyed by (turn_id, tool_call_id). Existing rows recover + # their turn through the stored parts; any row that cannot be + # matched is dropped rather than left able to collide. + if "turn_id" in db["tool_instantiations"].columns_dict: + return + db["tool_instantiations"].add_column("turn_id", str) + with db.conn: + db.execute(""" + update tool_instantiations set turn_id = ( + select turns.id from turns + join messages on messages.hash = turns.parent_message_hash + or messages.parent_hash = turns.parent_message_hash + join parts on parts.message_hash = messages.hash + where parts.type = 'tool_result' + and json_extract(parts.payload, '$.tool_call_id') + = tool_instantiations.tool_call_id + limit 1 + ) + """) + db.execute("delete from tool_instantiations where turn_id is null") + db["tool_instantiations"].transform(pk=("turn_id", "tool_call_id")) diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 6f251fd2c..c519b1a99 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -798,15 +798,48 @@ def __init__(self, path: str): ) response.text() response.log_to_db(store.db) + turn_id = next(iter(store.db["turns"].rows))["id"] assert list(store.db["tool_instantiations"].rows) == [ { "tool_call_id": "tc_1", "name": "Notes", "plugin": None, "arguments": '{"path": "/tmp/notes"}', + "turn_id": turn_id, } ] + def test_tool_instantiations_are_scoped_by_turn(self, store, mock_model): + # Providers with per-request counters can reuse a tool_call_id + # across turns - each turn keeps its own provenance row. + class Notes(llm.Toolbox): + def __init__(self, path: str): + self.path = path + + for path in ("/tmp/one", "/tmp/two"): + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "next", + tool_results=[ + llm.ToolResult( + name="Notes_read", + output="hello", + tool_call_id="call_0", + instance=Notes(path), + ) + ], + ) + response.text() + response.log_to_db(store.db) + arguments = { + row["turn_id"]: row["arguments"] + for row in store.db["tool_instantiations"].rows + } + assert sorted(arguments.values()) == [ + '{"path": "/tmp/one"}', + '{"path": "/tmp/two"}', + ] + def test_successive_library_turns_extend_the_thread(self, store, mock_model): conversation = mock_model.conversation() for reply in ("One", "Two"): From 3185ec353901ed05916fa59567bca25c95ca858c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 22:33:13 -0700 Subject: [PATCH 155/258] Settle concurrent message inserts with insert-or-ignore _ensure_message checked for the hash and then inserted, so two processes writing the same message could both see it absent and one would crash on the UNIQUE constraint. The insert is now insert-or-ignore inside the same transaction as the parts, and only the writer whose insert actually landed writes the parts. The count_where check stays as a fast path for the common already-stored case. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- llm/logs.py | 20 +++++++++++--------- tests/test_logs_store.py | 23 +++++++++++++++++++++++ 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/llm/logs.py b/llm/logs.py index a9abeb85a..f9cec6966 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -172,16 +172,18 @@ def _ensure_message( # everything below it is stored too. return hash with self.db.conn: - self.db["messages"].insert( - { - "hash": hash, - "parent_hash": parent_hash, - "role": message.role, - "provider_metadata": _dump(message.provider_metadata), - } + # Another writer can store the same hash between the check + # above and this insert. Insert-or-ignore settles who won, + # and only the winner writes the parts. + cursor = self.db.execute( + "insert or ignore into messages" + " (hash, parent_hash, role, provider_metadata)" + " values (?, ?, ?, ?)", + [hash, parent_hash, message.role, _dump(message.provider_metadata)], ) - for position, part in enumerate(message.parts): - self._write_part(hash, position, part, fragment_map) + if cursor.rowcount: + for position, part in enumerate(message.parts): + self._write_part(hash, position, part, fragment_map) return hash def _write_part( diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index c519b1a99..22ee0a25f 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -853,6 +853,29 @@ def test_successive_library_turns_extend_the_thread(self, store, mock_model): # ---- storage by reference -------------------------------------------- +class TestConcurrentWriters: + def test_losing_the_insert_race_neither_raises_nor_duplicates( + self, tmp_path, monkeypatch + ): + # Two connections to the same database. B checks for the hash + # while it is absent - simulated by disabling its fast-path + # check - then A wins the insert. B's own insert must quietly + # lose: no UNIQUE error, no second set of parts. + path = str(tmp_path / "logs.db") + store_a = LogStore(sqlite_utils.Database(path)) + store_b = LogStore(sqlite_utils.Database(path)) + message = llm.user("Hi") + tip = store_a.ensure_chain([message]) + monkeypatch.setattr( + sqlite_utils.db.Table, "count_where", lambda *args, **kwargs: 0 + ) + assert store_b.ensure_chain([message]) == tip + monkeypatch.undo() + assert store_a.db["messages"].count == 1 + assert store_a.db["parts"].count == 1 + assert store_a.verify() == [] + + class TestTurnInputBoundary: """A turn whose input ends [tool results, user prompt] owns both - the tool results must not vanish from display or the -T filter just From 42c70e05609e0f95fd33eb346a58fcfc37ee5402 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 22:34:47 -0700 Subject: [PATCH 156/258] Preserve repeated fragments in turn_fragments The primary key omitted the order column, so passing the same fragment twice collapsed to one provenance row - regressing the duplicate support m016 established for the legacy tables. Order joins the key, in the m024 create for new databases and via an m031 transform for existing ones. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- llm/migrations.py | 11 ++++++++++- tests/test_logs_store.py | 13 +++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/llm/migrations.py b/llm/migrations.py index 25dd9d8d7..fe71671d3 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -715,7 +715,7 @@ def m024_message_store_payloads(db): "order": int, "kind": str, # 'prompt' | 'system' }, - pk=("turn_id", "fragment_id", "kind"), + pk=("turn_id", "fragment_id", "kind", "order"), foreign_keys=( ("turn_id", "turns", "id"), ("fragment_id", "fragments", "id"), @@ -991,3 +991,12 @@ def m030_tool_instantiations_turn_scope(db): """) db.execute("delete from tool_instantiations where turn_id is null") db["tool_instantiations"].transform(pk=("turn_id", "tool_call_id")) + + +@migration +def m031_turn_fragments_order_pk(db): + # The same fragment can be passed to a prompt more than once - m016 + # established that for the legacy tables - so order joins the key + # and repeats are preserved instead of collapsing to one row. + if "order" not in db["turn_fragments"].pks: + db["turn_fragments"].transform(pk=("turn_id", "fragment_id", "kind", "order")) diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 22ee0a25f..976df247c 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -853,6 +853,19 @@ def test_successive_library_turns_extend_the_thread(self, store, mock_model): # ---- storage by reference -------------------------------------------- +class TestRepeatedFragments: + def test_passing_the_same_fragment_twice_keeps_both_rows(self, store, mock_model): + mock_model.enqueue(["ok"]) + response = mock_model.prompt("hi", fragments=["CONTEXT", "CONTEXT"]) + response.text() + response.log_to_db(store.db) + rows = list( + store.db["turn_fragments"].rows_where("kind = 'prompt'", order_by='"order"') + ) + assert [row["order"] for row in rows] == [0, 1] + assert rows[0]["fragment_id"] == rows[1]["fragment_id"] + + class TestConcurrentWriters: def test_losing_the_insert_race_neither_raises_nor_duplicates( self, tmp_path, monkeypatch From 0775cfa28e053f65ebe4a1242cd0f0650fcf49db Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 22:36:32 -0700 Subject: [PATCH 157/258] Resolve tool ids through each turn's turn_tools rows Tool ids for llm logs display came from a global name-to-id map, so when two content-addressed definitions shared a name, every turn reported whichever definition the scan visited last. The name-to-id map now comes from the turn's own turn_tools join - the same query that already produced the tools list - so each turn reports the definition it was actually given. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- llm/logs.py | 42 ++++++++++++++++++---------------------- tests/test_logs_store.py | 28 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 23 deletions(-) diff --git a/llm/logs.py b/llm/logs.py index f9cec6966..8be30ad0b 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -986,7 +986,25 @@ def log_row_extras(store: "LogStore", row: dict) -> dict: # `llm logs` is unchanged. call_ids = _part_ids(store, row.get("_tip_message_hash"), "tool_call") result_ids = _part_ids(store, row.get("_parent_message_hash"), "tool_result") - tool_ids = _tool_ids_by_name(store) + + # input_schema is rendered as a dict, so decode it here rather than + # handing the caller the raw JSON text out of the column. + tools = [ + {**tool_row, "input_schema": json.loads(tool_row["input_schema"] or "{}")} + for tool_row in store.db.query( + """ + select tools.id, tools.hash, tools.name, tools.description, + tools.input_schema + from tools join turn_tools on turn_tools.tool_id = tools.id + where turn_tools.turn_id = ? + """, + [row["id"]], + ) + ] + # Resolved through this turn's own turn_tools rows - definitions + # sharing a name can differ between turns, and a global map would + # attribute the wrong one. + tool_ids = {tool["name"]: tool["id"] for tool in tools} tool_calls = [ { @@ -1021,21 +1039,6 @@ def log_row_extras(store: "LogStore", row: dict) -> dict: for part in result_parts ] - # input_schema is rendered as a dict, so decode it here rather than - # handing the caller the raw JSON text out of the column. - tools = [ - {**tool_row, "input_schema": json.loads(tool_row["input_schema"] or "{}")} - for tool_row in store.db.query( - """ - select tools.id, tools.hash, tools.name, tools.description, - tools.input_schema - from tools join turn_tools on turn_tools.tool_id = tools.id - where turn_tools.turn_id = ? - """, - [row["id"]], - ) - ] - fragments: dict[str, list[dict]] = { "prompt_fragments": [], "system_fragments": [], @@ -1091,13 +1094,6 @@ def _part_ids(store: "LogStore", message_hash: str | None, type: str) -> dict: } -def _tool_ids_by_name(store: "LogStore") -> dict: - return { - tool_row["name"]: tool_row["id"] - for tool_row in store.db.query("select id, name from tools") - } - - def _instances_by_tool_call_id( store: "LogStore", turn_id: str, tool_call_ids: list ) -> dict: diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 976df247c..b2b51f5ae 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -853,6 +853,34 @@ def test_successive_library_turns_extend_the_thread(self, store, mock_model): # ---- storage by reference -------------------------------------------- +class TestPerTurnToolResolution: + def test_same_name_different_definitions_resolve_per_turn(self, store, mock_model): + # Two turns using tools that share a name but differ in + # definition - each turn's extras must report its own tool_id. + def make_tool(description): + return llm.Tool(name="lookup", description=description, input_schema={}) + + for description in ("first definition", "second definition"): + mock_model.enqueue(["ok"]) + response = mock_model.prompt("hi", tools=[make_tool(description)]) + response.text() + response.log_to_db(store.db) + + rows = merged_log_rows(store) + rows.reverse() + for row in rows: + extras = log_row_extras(store, row) + assert len(extras["tools"]) == 1 + descriptions_to_ids = { + log_row_extras(store, row)["tools"][0]["description"]: log_row_extras( + store, row + )["tools"][0]["id"] + for row in rows + } + assert len(descriptions_to_ids) == 2 + assert len(set(descriptions_to_ids.values())) == 2 + + class TestRepeatedFragments: def test_passing_the_same_fragment_twice_keeps_both_rows(self, store, mock_model): mock_model.enqueue(["ok"]) From bea75a1f3b7616292c678fc6bd32a656905e46ee Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2026 22:39:46 -0700 Subject: [PATCH 158/258] Document the search and tool_instantiations tables in the SQL schema The Cog-generated schema listing enumerated tables by hand and never picked up turn_search, its FTS index or tool_instantiations. They are in the list now, and the FTS note covers turn_search_fts alongside responses_fts. Also removes an import the completion-model change left unused. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- docs/logging.md | 26 ++++++++++++++++++++++++-- llm/default_plugins/openai_models.py | 2 +- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/docs/logging.md b/docs/logging.md index 808449821..5e656b1bd 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -674,6 +674,7 @@ for table in ( "tool_results_attachments", "messages", "parts", "part_attachments", "part_fragments", "threads", "turns", "turn_tools", "turn_fragments", + "turn_search", "turn_search_fts", "tool_instantiations", ): schema = db[table].schema cog.out(format(cleanup_sql(schema))) @@ -865,8 +866,29 @@ CREATE TABLE "turn_fragments" ( "kind" TEXT, PRIMARY KEY ("turn_id", "fragment_id", - "kind") + "kind", + "order") +); +CREATE TABLE "turn_search" ( + "id" INTEGER PRIMARY KEY, + "turn_id" TEXT REFERENCES "turns"("id"), + "prompt" TEXT, + "response" TEXT +); +CREATE VIRTUAL TABLE "turn_search_fts" USING FTS5 ( + "prompt", + "response", + content="turn_search" +); +CREATE TABLE "tool_instantiations" ( + "tool_call_id" TEXT, + "name" TEXT, + "plugin" TEXT, + "arguments" TEXT, + "turn_id" TEXT, + PRIMARY KEY ("turn_id", + "tool_call_id") ); ``` -`responses_fts` configures [SQLite full-text search](https://www.sqlite.org/fts5.html) against the `prompt` and `response` columns in the `responses` table. +`responses_fts` configures [SQLite full-text search](https://www.sqlite.org/fts5.html) against the `prompt` and `response` columns in the `responses` table. `turn_search_fts` does the same for the `turn_search` table, which holds the searchable text of each turn in the content-addressed tables - together these are what {ref}`llm logs -q ` queries. diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 635025761..f1d0124be 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -3,7 +3,7 @@ import os from collections.abc import AsyncGenerator, Iterable, Iterator from enum import Enum -from typing import Any, cast +from typing import Any import click import httpx From 947fc382586e42228b5afc972c7fe8fbcbdf2372 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 27 Jul 2026 12:47:45 -0700 Subject: [PATCH 159/258] Real transactions for message and turn writes `with db.conn:` is not a transaction under sqlite-utils - and worse, the sqlite3 context manager's exit commit will commit any transaction that happens to be open, which is how ensure_tool and ensure_fragment were breaking a wrapping transaction from the inside. A crash between the messages insert and its parts could strand a half-written message that the dedup check would then skip forever, and log() wrote turn, tools, fragments, provenance and search rows with no transaction at all. sqlite_transaction() delegates to sqlite-utils 4's Database.atomic() and mirrors its semantics - savepoint nesting included - on the 3.x floor, using the raw connection because routing BEGIN through db.execute trips sqlite-utils 4's bookkeeping, which commits it immediately. _ensure_message and the whole of log() run inside it: a failure anywhere rolls back everything and a retry starts clean. ensure_tool and ensure_fragment lose their committing wrappers and join whatever transaction is open; the fragment alias CLI commands get a real transaction in place of the placebo. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- llm/cli.py | 10 ++--- llm/logs.py | 14 +++++- llm/utils.py | 93 ++++++++++++++++++++++++++++------------ tests/test_logs_store.py | 48 +++++++++++++++++++++ 4 files changed, 130 insertions(+), 35 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index 52599352d..dd5919b10 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -86,6 +86,7 @@ resolve_schema_input, schema_dsl, schema_summary, + sqlite_transaction, token_usage_string, truncate_string, ) @@ -2942,9 +2943,9 @@ def fragments_set(alias, fragment): on conflict(alias) do update set fragment_id = excluded.fragment_id; """ - with db.conn: + with sqlite_transaction(db): fragment_id = ensure_fragment(db, resolved) - db.conn.execute(alias_sql, {"alias": alias, "fragment_id": fragment_id}) + db.execute(alias_sql, {"alias": alias, "fragment_id": fragment_id}) @fragments.command(name="show") @@ -2978,10 +2979,7 @@ def fragments_remove(alias): """ db = sqlite_utils.Database(logs_db_path()) migrate(db) - with db.conn: - db.conn.execute( - "delete from fragment_aliases where alias = :alias", {"alias": alias} - ) + db.execute("delete from fragment_aliases where alias = :alias", {"alias": alias}) @fragments.command(name="loaders") diff --git a/llm/logs.py b/llm/logs.py index 8be30ad0b..8578580a2 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -32,7 +32,13 @@ ToolCallPart, ToolResultPart, ) -from .utils import ensure_fragment, ensure_tool, make_schema_id, monotonic_ulid +from .utils import ( + ensure_fragment, + ensure_tool, + make_schema_id, + monotonic_ulid, + sqlite_transaction, +) __all__ = [ "HASH_PREFIX", @@ -171,7 +177,7 @@ def _ensure_message( # Already stored - and because the hash covers the parent, # everything below it is stored too. return hash - with self.db.conn: + with sqlite_transaction(self.db): # Another writer can store the same hash between the check # above and this insert. Insert-or-ignore settles who won, # and only the winner writes the parts. @@ -424,6 +430,10 @@ def log(self, response, thread_id: str | None = None) -> str: timings, usage, which model answered - goes on the turn, because message rows are shared and so cannot carry provenance. """ + with sqlite_transaction(self.db): + return self._log_in_transaction(response, thread_id) + + def _log_in_transaction(self, response, thread_id: str | None) -> str: if thread_id is None: conversation = getattr(response, "conversation", None) # A response logged outside any conversation still gets a diff --git a/llm/utils.py b/llm/utils.py index 86920837f..1db9cf136 100644 --- a/llm/utils.py +++ b/llm/utils.py @@ -7,6 +7,7 @@ import textwrap import threading import time +from contextlib import contextmanager from typing import Any, Final import click @@ -484,15 +485,10 @@ def ensure_fragment(db, content): source = None if isinstance(content, Fragment): source = content.source - with db.conn: - db.execute(sql, {"hash": hash_id, "content": content, "source": source}) - return next( - iter( - db.query( - "select id from fragments where hash = :hash", {"hash": hash_id} - ) - ) - )["id"] + db.execute(sql, {"hash": hash_id, "content": content, "source": source}) + return db.execute( + "select id from fragments where hash = :hash", {"hash": hash_id} + ).fetchone()[0] def ensure_tool(db, tool): @@ -501,24 +497,21 @@ def ensure_tool(db, tool): values (:hash, :name, :description, :input_schema, :plugin) on conflict(hash) do nothing """ - with db.conn: - db.execute( - sql, - { - "hash": tool.hash(), - "name": tool.name, - "description": tool.description, - "input_schema": json.dumps(tool.input_schema), - "plugin": tool.plugin, - }, - ) - return next( - iter( - db.query( - "select id from tools where hash = :hash", {"hash": tool.hash()} - ) - ) - )["id"] + # No `with db.conn:` here - its exit commit would also commit any + # open outer transaction, such as the one wrapping a turn write. + db.execute( + sql, + { + "hash": tool.hash(), + "name": tool.name, + "description": tool.description, + "input_schema": json.dumps(tool.input_schema), + "plugin": tool.plugin, + }, + ) + return db.execute( + "select id from tools where hash = :hash", {"hash": tool.hash()} + ).fetchone()[0] def maybe_fenced_code(content: str) -> str: @@ -740,3 +733,49 @@ def _fresh(ms: int) -> bytes: timestamp = int.to_bytes(ms, TIMESTAMP_LEN, "big") randomness = os.urandom(RANDOMNESS_LEN) return timestamp + randomness + + +@contextmanager +def sqlite_transaction(db): + """A real transaction over a sqlite_utils Database. + + ``with db.conn:`` is not one - sqlite-utils effectively runs in + autocommit, each helper call commits itself and the context manager + has nothing left to roll back. + + sqlite-utils 4 provides Database.atomic() with the semantics needed + here; the supported floor includes sqlite-utils 3.x, which does + not, so this mirrors it there. The raw connection is used on + purpose: routing BEGIN through db.execute trips sqlite-utils 4's + transaction bookkeeping, which commits it immediately. + """ + if hasattr(db, "atomic"): + with db.atomic(): + yield + return + conn = db.conn + if conn.in_transaction: + # Nested use: a savepoint, so an inner failure rolls back only + # the inner block, matching Database.atomic(). + savepoint = f"llm_txn_{os.urandom(8).hex()}" + conn.execute(f"SAVEPOINT {savepoint}") + try: + yield + except BaseException: + if conn.in_transaction: + conn.execute(f"ROLLBACK TO SAVEPOINT {savepoint}") + conn.execute(f"RELEASE SAVEPOINT {savepoint}") + raise + else: + conn.execute(f"RELEASE SAVEPOINT {savepoint}") + else: + conn.execute("BEGIN") + try: + yield + except BaseException: + if conn.in_transaction: + conn.execute("ROLLBACK") + raise + else: + if conn.in_transaction: + conn.execute("COMMIT") diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index b2b51f5ae..c12a14401 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -7,6 +7,7 @@ """ import json +import sqlite3 import pytest import sqlite_utils @@ -894,6 +895,53 @@ def test_passing_the_same_fragment_twice_keeps_both_rows(self, store, mock_model assert rows[0]["fragment_id"] == rows[1]["fragment_id"] +class TestAtomicWrites: + """sqlite-utils runs in autocommit, so `with db.conn` was never a + transaction - a crash mid-write could strand a message without its + parts, and the dedup check would then skip it forever.""" + + def test_failed_part_write_rolls_back_the_message(self, store, monkeypatch): + message = Message(role="user", parts=[TextPart(text="a"), TextPart(text="b")]) + original = LogStore._write_part + + def flaky(self, hash_, position, part, fragment_map): + if position == 1: + raise RuntimeError("disk full") + return original(self, hash_, position, part, fragment_map) + + monkeypatch.setattr(LogStore, "_write_part", flaky) + with pytest.raises(RuntimeError): + store.ensure_chain([message]) + monkeypatch.undo() + assert store.db["messages"].count == 0 + assert store.db["parts"].count == 0 + # A retry can now write the whole message + tip = store.ensure_chain([message]) + assert store.load_chain(tip) == [message] + assert store.verify() == [] + + def test_failed_turn_write_rolls_back_the_whole_turn( + self, store, mock_model, monkeypatch + ): + mock_model.enqueue(["ok"]) + response = mock_model.prompt("Hi") + response.text() + # Fail at the search refresh, one of the last steps of log() + monkeypatch.setattr( + "llm.logs.TURN_SEARCH_INSERT_SQL", "this is not sql {turn_filter}" + ) + with pytest.raises(sqlite3.OperationalError): + store.log(response) + monkeypatch.undo() + assert store.db["turns"].count == 0 + assert store.db["messages"].count == 0 + assert store.db["threads"].count == 0 + # And the retry writes everything + store.log(response) + assert store.db["turns"].count == 1 + assert store.verify() == [] + + class TestConcurrentWriters: def test_losing_the_insert_race_neither_raises_nor_duplicates( self, tmp_path, monkeypatch From ceba6bec7c4e7769c2cfbed9be883cf45db7524e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 27 Jul 2026 12:59:33 -0700 Subject: [PATCH 160/258] Migrations run in real transactions, m029 defers foreign keys The with db.conn blocks in m027, m029 and m030 were the same autocommit placebo as the write paths: an interrupted m029 could leave dangling parts and parent hashes mid-remap, unable to reconstruct the missing link on retry. All three now run inside sqlite_transaction, and m029 additionally sets PRAGMA defer_foreign_keys for the duration of its transaction - rewriting messages.hash while parts and turns still reference it fails immediately on connections that enforce foreign keys, and deferral lets the whole remap settle at commit. Verified with foreign_keys=ON end to end, including a clean foreign_key_check afterwards. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- llm/migrations.py | 13 ++++++++++--- tests/test_logs_store.py | 42 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/llm/migrations.py b/llm/migrations.py index fe71671d3..5600bd67b 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -2,6 +2,8 @@ import json from collections.abc import Callable +from .utils import sqlite_transaction + MIGRATIONS: list[Callable] = [] migration = MIGRATIONS.append @@ -828,7 +830,7 @@ def m027_parts_text_column(db): # the old keys - so an interrupted run can be retried without the # already-migrated rows (whose payloads no longer have a text key) # being blanked back to text=None. - with db.conn: + with sqlite_transaction(db): for row in list(db.query("select id, type, payload from parts")): payload = json.loads(row["payload"]) if row["payload"] else {} if "type" not in payload and "text" not in payload: @@ -903,7 +905,12 @@ def m029_rehash_messages(db): mapping[old_hash] = message_hash(message, mapping.get(parent, parent)) queue.extend(children.get(old_hash, [])) - with db.conn: + with sqlite_transaction(db): + # Primary keys are rewritten while other rows still reference + # them; defer enforcement to commit for connections that run + # with foreign keys enabled. The pragma only takes effect + # inside a transaction and resets itself at commit. + db.conn.execute("PRAGMA defer_foreign_keys = ON") seen: set = set() for old_hash, new_hash in mapping.items(): if new_hash in seen: @@ -976,7 +983,7 @@ def m030_tool_instantiations_turn_scope(db): if "turn_id" in db["tool_instantiations"].columns_dict: return db["tool_instantiations"].add_column("turn_id", str) - with db.conn: + with sqlite_transaction(db): db.execute(""" update tool_instantiations set turn_id = ( select turns.id from turns diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index c12a14401..0d7d4ff8d 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -1097,6 +1097,48 @@ def test_attachment_chain_verifies(self, store, tmp_path): ) assert store.verify() == [] + def test_m029_survives_foreign_key_enforcement(self, tmp_path): + # Rekeying messages.hash while parts and turns still reference + # it would fail immediately on a connection running with + # PRAGMA foreign_keys = ON - the migration defers enforcement + # to commit, inside a real transaction. + db = sqlite_utils.Database(str(tmp_path / "fk.db")) + db.conn.execute("PRAGMA foreign_keys = ON") + store = LogStore(db) + path = tmp_path / "x.png" + path.write_bytes(b"PNG BYTES") + messages = [ + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type="image/png", path=str(path)) + ) + ], + ), + llm.assistant("A fine image"), + ] + tip = store.ensure_chain(messages) + real = [row["hash"] for row in db.query("select hash from messages")] + fakes = {h: "b2:" + format(i, "032x") for i, h in enumerate(real)} + db.conn.execute("PRAGMA foreign_keys = OFF") + for old, fake in fakes.items(): + db.execute("update messages set hash = ? where hash = ?", [fake, old]) + db.execute( + "update messages set parent_hash = ? where parent_hash = ?", + [fake, old], + ) + db.execute( + "update parts set message_hash = ? where message_hash = ?", + [fake, old], + ) + db.execute("delete from _llm_migrations where name = 'm029_rehash_messages'") + db.conn.execute("PRAGMA foreign_keys = ON") + migrate(db) + assert store.verify() == [] + assert store.load_chain(tip) == messages + assert db.conn.execute("PRAGMA foreign_key_check").fetchall() == [] + def test_m029_recomputes_stale_hashes(self, store, tmp_path): # Build a real chain, then rewrite its hashes to bogus values - # simulating rows written by the old path-based algorithm - and From 1b474a649c7f77e3b205a8dcc9ae3b538e0ba4e1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 27 Jul 2026 13:02:27 -0700 Subject: [PATCH 161/258] Attachment identity: real bytes plus media type, drift detected The canonical attachment form trusted Attachment.id(), whose cached value meant verify() re-derived hashes from a remembered answer rather than the file - a path-backed attachment could change on disk and the log would keep vouching for it. It also omitted the media type, which the model sees: identical bytes logged as image/png and text/plain collapsed to one message, with the second reload wearing the first's type. The canonical form now recomputes the content hash from the actual bytes every time and includes the resolved type. Path-backed attachments still store no copy of their bytes - by choice, logs.db does not swallow large media - so their fidelity depends on the file: verify() re-reads it and reports a changed or deleted file as a broken hash instead of passing silently. m029's remap body becomes a shared _rehash_messages helper and m032 reruns it, since type joining the canonical form changes attachment-bearing hashes again. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- docs/logging.md | 2 +- llm/logs.py | 41 +++++++++++---- llm/migrations.py | 31 +++++++++--- tests/test_logs_store.py | 105 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 162 insertions(+), 17 deletions(-) diff --git a/docs/logging.md b/docs/logging.md index 5e656b1bd..7ecf00f21 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -441,7 +441,7 @@ The `b2:` prefix names the algorithm that produced the hash, so any future chang Two design decisions matter here: -- **The hash covers resolved content.** {ref}`Fragment ` references are expanded to their full text before hashing, and attachments are represented by the SHA-256 hash of their bytes - never by the filesystem path they were loaded from, so editing a file after logging cannot leave a stale hash looking valid. An attachment supplied as a URL is hashed by that URL: the log records which URL was sent, not whatever it served that day. The internal `LogStore.verify()` method re-derives every hash from the stored rows to check all of this stays true. +- **The hash covers resolved content.** {ref}`Fragment ` references are expanded to their full text before hashing, and attachments are represented by the SHA-256 hash of their bytes together with their media type - the model sees the type, so identical bytes sent as `image/png` and as `text/plain` are different requests. An attachment supplied as a URL is hashed by that URL: the log records which URL was sent, not whatever it served that day. Attachments loaded from a filesystem path are stored by reference to that file rather than copied into the database, so their fidelity depends on the file staying put - `LogStore.verify()` re-reads the actual bytes when it re-derives every hash, and reports a changed or deleted file as a broken hash rather than letting it pass silently. - **The parent hash participates in the hash.** The same content appearing at a different point in a conversation is a different node. This is what makes two conversations that share a prefix collapse to shared rows, with no explicit comparison required: replaying the same messages produces the same hashes, and a hash that is already present needs nothing written. A consequence of the second decision is that a stateless client - one that holds its own conversation history and re-sends the whole thing with every call - writes only the new tail on each request. It also makes forks cheap, as described next. Here the first conversation from the worked example is logged again, followed by a second conversation that starts with the same prompt and then diverges: diff --git a/llm/logs.py b/llm/logs.py index 8578580a2..b2460c5fa 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -81,16 +81,39 @@ def content_hash(obj: Any) -> str: def _canonical_attachment(attachment) -> dict: - """The hashed form of an attachment: its content id. - - Identity is the sha256 of the bytes - the same id that keys the - attachments table - never the filesystem path they happened to live - at, so editing a file after logging cannot leave a stale hash - looking valid, and the same bytes at two paths are one identity. - URL attachments hash the URL itself: the log records which URL was - sent, not whatever that URL served on the day. + """The hashed form of an attachment: content identity plus the + model-visible media type. + + The content hash is recomputed from the actual bytes every time, + never taken from Attachment.id()'s cache - that is what lets + verify() notice that a path-backed file changed after logging. + Path-backed attachments keep no copy of their bytes in the store + (by design: logs.db does not swallow large media), so their + fidelity depends on the file staying put; a changed or deleted + file shows up as a broken hash rather than passing silently. + + The type participates because the model sees it: identical bytes + sent as image/png and as text/plain are different requests. URL + attachments hash the URL itself - the log records which URL was + sent, not whatever it served that day. """ - return {"id": attachment.id()} + if attachment.content: + content_id = hashlib.sha256(attachment.content).hexdigest() + elif attachment.path: + try: + with open(attachment.path, "rb") as fp: + content_id = hashlib.sha256(fp.read()).hexdigest() + except OSError: + content_id = f"missing:{attachment.path}" + else: + content_id = hashlib.sha256( + json.dumps({"url": attachment.url}).encode("utf-8") + ).hexdigest() + try: + type_ = attachment.resolve_type() + except Exception: + type_ = attachment.type + return {"id": content_id, "type": type_} def message_hash(message: Message, parent_hash: str | None) -> str: diff --git a/llm/migrations.py b/llm/migrations.py index 5600bd67b..071b700e0 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -863,13 +863,12 @@ def m028_tool_instantiations(db): ) -@migration -def m029_rehash_messages(db): - # Message hashes now identify attachments by the sha256 of their - # content rather than the filesystem path they were loaded from. - # Recompute every stored hash from resolved content, bottom-up, and - # repoint everything that references one. Hashing from content also - # merges messages that only ever differed by attachment path. +def _rehash_messages(db): + # Recompute every stored message hash from resolved content, + # bottom-up, and repoint everything that references one. Run by any + # migration that changes what participates in the hash; hashing + # from content also merges messages whose old hashes only ever + # differed by details the hash no longer covers. if not db["messages"].exists() or not db["messages"].count: return # Runtime import - llm.logs imports this module at import time, but @@ -973,6 +972,14 @@ def _load_json(value): return json.loads(value) if value else None +@migration +def m029_rehash_messages(db): + # Message hashes began identifying attachments by the sha256 of + # their content rather than the filesystem path they were loaded + # from. + _rehash_messages(db) + + @migration def m030_tool_instantiations_turn_scope(db): # tool_call_id is not globally unique - providers with per-request @@ -1007,3 +1014,13 @@ def m031_turn_fragments_order_pk(db): # and repeats are preserved instead of collapsing to one row. if "order" not in db["turn_fragments"].pks: db["turn_fragments"].transform(pk=("turn_id", "fragment_id", "kind", "order")) + + +@migration +def m032_rehash_for_attachment_types(db): + # The canonical attachment form now includes the media type - the + # model sees it, so identical bytes sent as different types are + # different requests - and the content hash is recomputed from the + # actual bytes rather than a cached id. Both change hashes of + # attachment-bearing messages, so recompute the stored tree again. + _rehash_messages(db) diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 0d7d4ff8d..cd50225f7 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -1097,6 +1097,111 @@ def test_attachment_chain_verifies(self, store, tmp_path): ) assert store.verify() == [] + def test_media_type_participates_in_identity(self, tmp_path): + path = tmp_path / "x.bin" + path.write_bytes(b"SAME BYTES") + + def hash_as(type_): + return message_hash( + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type=type_, path=str(path)) + ) + ], + ), + None, + ) + + # The model sees the media type: identical bytes sent as + # different types are different requests. + assert hash_as("image/png") != hash_as("text/plain") + + def test_cached_attachment_id_is_not_trusted(self, tmp_path): + path = tmp_path / "x.png" + path.write_bytes(b"first") + attachment = Attachment(type="image/png", path=str(path)) + attachment.id() # caches _id from the current bytes + message = Message(role="user", parts=[AttachmentPart(attachment=attachment)]) + first = message_hash(message, None) + path.write_bytes(b"second") + assert message_hash(message, None) != first + + def test_editing_the_file_after_logging_breaks_verify(self, store, tmp_path): + path = tmp_path / "x.png" + path.write_bytes(b"ORIGINAL") + tip = store.ensure_chain( + [ + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type="image/png", path=str(path)) + ) + ], + ) + ] + ) + assert store.verify() == [] + path.write_bytes(b"TAMPERED") + assert store.verify() == [tip] + + def test_deleting_the_file_is_detected_not_fatal(self, store, tmp_path): + path = tmp_path / "x.png" + path.write_bytes(b"ORIGINAL") + tip = store.ensure_chain( + [ + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type="image/png", path=str(path)) + ) + ], + ) + ] + ) + path.unlink() + assert store.verify() == [tip] + + def test_m032_reruns_the_rehash(self, store, tmp_path): + path = tmp_path / "x.png" + path.write_bytes(b"PNG BYTES") + messages = [ + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type="image/png", path=str(path)) + ) + ], + ), + llm.assistant("A fine image"), + ] + tip = store.ensure_chain(messages) + db = store.db + real = [row["hash"] for row in db.query("select hash from messages")] + fakes = {h: "b2:" + format(i, "032x") for i, h in enumerate(real)} + with db.conn: + for old, fake in fakes.items(): + db.execute("update messages set hash = ? where hash = ?", [fake, old]) + db.execute( + "update messages set parent_hash = ? where parent_hash = ?", + [fake, old], + ) + db.execute( + "update parts set message_hash = ? where message_hash = ?", + [fake, old], + ) + db.execute( + "delete from _llm_migrations where name = 'm032_rehash_for_attachment_types'" + ) + assert store.verify() != [] + migrate(db) + assert store.verify() == [] + assert store.load_chain(tip) == messages + def test_m029_survives_foreign_key_enforcement(self, tmp_path): # Rekeying messages.hash while parts and turns still reference # it would fail immediately on a connection running with From 687dd04c521453cc5a4aa0345bd3b811b7b68776 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 27 Jul 2026 13:04:38 -0700 Subject: [PATCH 162/258] m030 traverses the whole input segment before deleting The turn-id backfill matched the turn's parent message and the parent's children - the wrong direction. A tool result followed by a fresh user prompt lives in the parent's own parent, so those rows matched nothing and the unmatched-row delete silently discarded valid provenance. The query now checks the parent and, when the parent is a user message, its parent. Databases that already ran the broken backfill have lost those rows; this fixes every database that has not. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- llm/migrations.py | 14 ++++++++++---- tests/test_logs_store.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/llm/migrations.py b/llm/migrations.py index 071b700e0..56781dcad 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -991,15 +991,21 @@ def m030_tool_instantiations_turn_scope(db): return db["tool_instantiations"].add_column("turn_id", str) with sqlite_transaction(db): + # The tool result sits either in the turn's parent message, + # or - when a fresh user prompt followed the results - in the + # parent's own parent, one step up the same input segment. db.execute(""" update tool_instantiations set turn_id = ( select turns.id from turns - join messages on messages.hash = turns.parent_message_hash - or messages.parent_hash = turns.parent_message_hash - join parts on parts.message_hash = messages.hash - where parts.type = 'tool_result' + join parts on parts.type = 'tool_result' and json_extract(parts.payload, '$.tool_call_id') = tool_instantiations.tool_call_id + where parts.message_hash = turns.parent_message_hash + or parts.message_hash = ( + select messages.parent_hash from messages + where messages.hash = turns.parent_message_hash + and messages.role = 'user' + ) limit 1 ) """) diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index cd50225f7..96cef5bee 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -993,6 +993,42 @@ def test_tool_filters_match(self, store, mock_model): assert len(merged_log_rows(store, tool_names=["t"])) == 1 assert merged_log_rows(store, tool_names=["other"]) == [] + def test_m030_backfill_finds_results_behind_a_user_prompt(self, store, mock_model): + # A turn whose input ends [tool result, user prompt] keeps its + # provenance row through the m030 backfill - the result lives + # in the parent's parent, and a wrong-direction traversal used + # to delete these rows as unmatched. + class Notes(llm.Toolbox): + def __init__(self, path: str): + self.path = path + + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "next question", + messages=[llm.user("orig"), llm.assistant("first answer")], + tool_results=[ + llm.ToolResult( + name="Notes_read", + output="RESULT", + tool_call_id="c9", + instance=Notes("/tmp/n"), + ) + ], + ) + response.text() + response.log_to_db(store.db) + db = store.db + turn_id = next(iter(db["turns"].rows))["id"] + # Rewind the table to its pre-m030 shape + db["tool_instantiations"].transform(pk="tool_call_id", drop={"turn_id"}) + db.execute( + "delete from _llm_migrations" + " where name = 'm030_tool_instantiations_turn_scope'" + ) + migrate(db) + rows = list(db["tool_instantiations"].rows) + assert [row["turn_id"] for row in rows] == [turn_id] + class TestMigrationRetrySafety: def test_m027_retry_does_not_erase_migrated_text(self, store, mock_model): From e7cb0233ea4cd9c3c79eda9045a27aa7e3aa3d52 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 27 Jul 2026 13:05:54 -0700 Subject: [PATCH 163/258] Repeated attachments keep every junction row part_attachments left order out of its primary key, so a tool result carrying the same attachment twice raised a UNIQUE error while the payload recorded both references - the two representations could not agree. Order joins the key, in the m024 create for new databases and an m033 transform for existing ones, matching part_fragments and turn_fragments. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- llm/migrations.py | 14 ++++++++++++-- tests/test_logs_store.py | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/llm/migrations.py b/llm/migrations.py index 56781dcad..35219c83b 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -492,7 +492,7 @@ def m023_content_addressed_messages(db): "attachment_id": str, "order": int, }, - pk=("part_id", "attachment_id"), + pk=("part_id", "attachment_id", "order"), foreign_keys=( ("part_id", "parts", "id"), ("attachment_id", "attachments", "id"), @@ -634,7 +634,7 @@ def m024_message_store_payloads(db): "attachment_id": str, "order": int, }, - pk=("part_id", "attachment_id"), + pk=("part_id", "attachment_id", "order"), foreign_keys=( ("part_id", "parts", "id"), ("attachment_id", "attachments", "id"), @@ -1030,3 +1030,13 @@ def m032_rehash_for_attachment_types(db): # actual bytes rather than a cached id. Both change hashes of # attachment-bearing messages, so recompute the stored tree again. _rehash_messages(db) + + +@migration +def m033_part_attachments_order_pk(db): + # A tool result can return the same attachment more than once; with + # order outside the key the second reference raised a UNIQUE error + # while the payload happily recorded both. Order joins the key, + # matching part_fragments and turn_fragments. + if "order" not in db["part_attachments"].pks: + db["part_attachments"].transform(pk=("part_id", "attachment_id", "order")) diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 96cef5bee..7c64ec37d 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -1328,6 +1328,27 @@ def test_m029_recomputes_stale_hashes(self, store, tmp_path): assert store.load_chain(tip) == messages +class TestRepeatedAttachments: + def test_a_tool_result_can_carry_the_same_attachment_twice(self, store): + attachment = Attachment(type="image/png", content=b"PNG BYTES") + message = Message( + role="tool", + parts=[ + ToolResultPart( + name="t", + output="ok", + tool_call_id="c1", + attachments=[attachment, attachment], + ) + ], + ) + tip = store.ensure_chain([message]) + assert store.db["part_attachments"].count == 2 + loaded = store.load_chain(tip) + assert len(loaded[0].parts[0].attachments) == 2 + assert store.verify() == [] + + class TestPartStorageFormat: """Literal text is stored raw in its own column - never escaped, never parsed - and the JSON payload holds only structure, with the From a1c2f4b2b35121eb837e6588fd33b7f6a9df44cb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 27 Jul 2026 13:07:42 -0700 Subject: [PATCH 164/258] Resolve tool-result row ids across the turn's input segment The input-boundary fix taught display to find tool results one message above the parent, but _part_ids still queried the parent alone, so those results rendered with id=null despite having a stored parts row. The row builder now records the segment's message hashes - the same trailing user/tool run it derives for display - and part id resolution queries all of them. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- llm/cli.py | 1 + llm/logs.py | 42 +++++++++++++++++++++++++++++++++------- tests/test_logs_store.py | 3 +++ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index dd5919b10..70731007a 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -1610,6 +1610,7 @@ def annotate_log_rows(db, rows, expand=False, truncate=False): "_input_parts", "_output_parts", "_parent_message_hash", + "_input_message_hashes", "_tip_message_hash", "_legacy", "_search_rank", diff --git a/llm/logs.py b/llm/logs.py index b2460c5fa..feba07165 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -886,11 +886,36 @@ def build(self, row: dict) -> dict: # Internal, stripped before rendering - the enrichment # needs them to find the parts rows behind these parts. "_parent_message_hash": row["parent_message_hash"], + "_input_message_hashes": self._input_segment_hashes( + row["parent_message_hash"] + ), "_tip_message_hash": row["tip_message_hash"], } ) return built + def _input_segment_hashes(self, parent_hash: str | None) -> list[str]: + """Hashes of the turn's own input messages - the same trailing + user/tool run build() derives, walked directly in the table.""" + hashes: list[str] = [] + hash_ = parent_hash + while hash_: + message_row = next( + iter( + self.store.db.query( + "select parent_hash, role from messages where hash = ?", + [hash_], + ) + ), + None, + ) + if message_row is None or message_row["role"] not in ("user", "tool"): + break + hashes.append(hash_) + hash_ = message_row["parent_hash"] + hashes.reverse() + return hashes + def log_rows( store: "LogStore", @@ -1017,8 +1042,8 @@ def log_row_extras(store: "LogStore", row: dict) -> dict: # parts.id and tools.id stand in for the row ids the old # tool_calls / tool_results tables exposed, so the JSON shape of # `llm logs` is unchanged. - call_ids = _part_ids(store, row.get("_tip_message_hash"), "tool_call") - result_ids = _part_ids(store, row.get("_parent_message_hash"), "tool_result") + call_ids = _part_ids(store, [row.get("_tip_message_hash")], "tool_call") + result_ids = _part_ids(store, row.get("_input_message_hashes") or [], "tool_result") # input_schema is rendered as a dict, so decode it here rather than # handing the caller the raw JSON text out of the column. @@ -1114,15 +1139,18 @@ def _attachment_summary(attachment) -> dict: } -def _part_ids(store: "LogStore", message_hash: str | None, type: str) -> dict: - "Map tool_call_id to the parts row id, for one message." - if not message_hash: +def _part_ids(store: "LogStore", message_hashes: list, type: str) -> dict: + "Map tool_call_id to the parts row id, across the given messages." + message_hashes = [hash_ for hash_ in message_hashes if hash_] + if not message_hashes: return {} + placeholders = ",".join("?" * len(message_hashes)) return { json.loads(part_row["payload"]).get("tool_call_id"): part_row["id"] for part_row in store.db.query( - "select id, payload from parts where message_hash = ? and type = ?", - [message_hash, type], + f"select id, payload from parts" + f" where message_hash in ({placeholders}) and type = ?", + message_hashes + [type], ) } diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 7c64ec37d..7622fbfad 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -986,6 +986,9 @@ def test_tool_results_and_prompt_both_display(self, store, mock_model): assert row["prompt"] == "next question" extras = log_row_extras(store, row) assert [result["output"] for result in extras["tool_results"]] == ["RESULT"] + # The parts row id resolves even though the result sits one + # message above the parent. + assert extras["tool_results"][0]["id"] is not None def test_tool_filters_match(self, store, mock_model): self._log_turn_with_results_and_prompt(store, mock_model) From 7805f45e32796e8df0352b74a0b6c7332fc1ae39 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 27 Jul 2026 13:09:07 -0700 Subject: [PATCH 165/258] Document tool_instantiations' compound key The table-by-table entry still described the pre-m030 single-column key; it is keyed by (turn_id, tool_call_id), with a note on why - provider-supplied call ids are not guaranteed unique across turns. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- docs/logging.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/logging.md b/docs/logging.md index 7ecf00f21..9e7be1104 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -544,7 +544,7 @@ The full schema for these tables appears in {ref}`the SQL schema section ` definitions were available to a turn, referencing the `tools` table. - `turn_fragments` - which fragments a turn was given, with their `kind` (`prompt` or `system`) and order. Provenance lives here rather than on the shared message rows, and this table is what powers `llm logs -f`. - `turn_search` - the searchable text of each turn: the literal prompt the user typed (fragment content excluded, via the `text_ref` literals described above) and the assistant's text output. An FTS5 index over this table, `turn_search_fts`, is what powers {ref}`llm logs -q `. Derived from the stored parts when a turn is logged; a turn with no prompt or response text, such as a pure tool call, gets no row. -- `tool_instantiations` - which configured {ref}`toolbox ` instance served a tool call: the toolbox name, its plugin and its constructor arguments, keyed by `tool_call_id`. Message rows are shared between conversations and so cannot carry this kind of local execution provenance; this table joins to the chain from outside it, and is what lets `llm logs` show that a `SQLite_query` call ran against `SQLite("mydb.db")`. +- `tool_instantiations` - which configured {ref}`toolbox ` instance served a tool call: the toolbox name, its plugin and its constructor arguments, keyed by `(turn_id, tool_call_id)` - call ids supplied by providers are not guaranteed unique across turns. Message rows are shared between conversations and so cannot carry this kind of local execution provenance; this table joins to the chain from outside it, and is what lets `llm logs` show that a `SQLite_query` call ran against `SQLite("mydb.db")`. (logging-message-store-queries)= @@ -821,7 +821,8 @@ CREATE TABLE "part_attachments" ( "attachment_id" TEXT REFERENCES "attachments"("id"), "order" INTEGER, PRIMARY KEY ("part_id", - "attachment_id") + "attachment_id", + "order") ); CREATE TABLE "part_fragments" ( "part_id" INTEGER REFERENCES "parts"("id"), From 200153d0a69035e3febc0c14a016565e160aee0f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 27 Jul 2026 13:10:40 -0700 Subject: [PATCH 166/258] Catch only OSError when resolving a missing attachment's type Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- llm/logs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llm/logs.py b/llm/logs.py index feba07165..c70b93960 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -111,7 +111,9 @@ def _canonical_attachment(attachment) -> dict: ).hexdigest() try: type_ = attachment.resolve_type() - except Exception: + except OSError: + # A deleted path-backed file - the content hash above already + # carries the missing marker type_ = attachment.type return {"id": content_id, "type": type_} From 87636ccd1caed5b9b6581bb7057bb33bc8e9e2b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 28 Jul 2026 16:44:34 -0700 Subject: [PATCH 167/258] One migration creates the message store, in final form The branch had accumulated eleven migrations - m023 through m033 - whose only purpose was upgrading data written by earlier commits of this same unreleased branch: payload rewrites, two full hash recomputations, provenance backfills, primary key transforms. Nobody has that data, every future user would have carried the machinery forever, and five of the ten code review findings so far were bugs in exactly these migrations. They are replaced by a single m023_message_store that creates every table in its final shape: parts with the text column, order in the turn_fragments and part_attachments keys, tool_instantiations keyed by (turn_id, tool_call_id), turn_search with its FTS index. Nothing is dropped, migrated or backfilled - these tables have never existed in any released version of LLM (0.32a3 ends at m022), so a database either gains them fresh or is in an unsupported development state, in which case the creates fail loudly rather than touching what is there. A test pins that failure mode. This also resolves two of the three open review findings by deletion: the m032 rehash that would have blessed pre-upgrade attachment drift, and the m030 retry hole - both were migrations for data that should never have had migrations. Refs https://github.com/simonw/llm/pull/1562#issuecomment-5087484395 Co-Authored-By: Claude Fable 5 --- docs/logging.md | 2 +- llm/migrations.py | 523 ++++++--------------------------------- tests/test_logs_store.py | 192 +------------- 3 files changed, 92 insertions(+), 625 deletions(-) diff --git a/docs/logging.md b/docs/logging.md index 9e7be1104..27379c5f4 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -882,11 +882,11 @@ CREATE VIRTUAL TABLE "turn_search_fts" USING FTS5 ( content="turn_search" ); CREATE TABLE "tool_instantiations" ( + "turn_id" TEXT REFERENCES "turns"("id"), "tool_call_id" TEXT, "name" TEXT, "plugin" TEXT, "arguments" TEXT, - "turn_id" TEXT, PRIMARY KEY ("turn_id", "tool_call_id") ); diff --git a/llm/migrations.py b/llm/migrations.py index 35219c83b..bc6b44dfa 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -1,9 +1,6 @@ import datetime -import json from collections.abc import Callable -from .utils import sqlite_transaction - MIGRATIONS: list[Callable] = [] migration = MIGRATIONS.append @@ -431,160 +428,78 @@ def m022_response_reasoning(db): db["responses"].add_column("reasoning", str) -@migration -def m023_content_addressed_messages(db): - # The content-addressed message tree. A message's hash covers its own - # content *and* its parent's hash, so conversations sharing a prefix - # share the rows storing it. Nothing here replaces the older tables - - # they stay exactly as they are so existing logs need no backfill. - db["messages"].create( - { - "hash": str, - "parent_hash": str, - "role": str, - "provider_metadata": str, - }, - pk="hash", - foreign_keys=(("parent_hash", "messages", "hash"),), - ) - db["messages"].create_index(["parent_hash"]) - - # Parts are plain child rows rather than content-addressed in their - # own right: prefix sharing already dedupes at the message level, and - # the genuinely large payloads (attachments, fragments) live in - # tables that are content-addressed already. - db["parts"].create( - { - "id": int, - "message_hash": str, - "position": int, - "type": str, - # text / reasoning - "text": str, - "fragment_id": int, - "redacted": int, - # tool_call / tool_result - "name": str, - "arguments": str, - "output": str, - "tool_call_id": str, - "server_executed": int, - "exception": str, - "tool_id": int, - "instance_id": int, - "provider_metadata": str, - }, - pk="id", - foreign_keys=( - ("message_hash", "messages", "hash"), - ("fragment_id", "fragments", "id"), - ("tool_id", "tools", "id"), - ("instance_id", "tool_instances", "id"), - ), - ) - db["parts"].create_index(["message_hash", "position"], unique=True) - - # Covers both AttachmentPart and the attachments a tool result can - # carry, so there is one mechanism instead of two. - db["part_attachments"].create( - { - "part_id": int, - "attachment_id": str, - "order": int, - }, - pk=("part_id", "attachment_id", "order"), - foreign_keys=( - ("part_id", "parts", "id"), - ("attachment_id", "attachments", "id"), - ), - ) - - # A turn is one call to a model. Provenance lives here rather than on - # the message rows, which are shared and so cannot carry it. - db["turns"].create( - { - "id": str, - "thread_id": str, - "parent_message_hash": str, - "tip_message_hash": str, - "model": str, - "resolved_model": str, - "options_json": str, - "schema_id": str, - "input_tokens": int, - "output_tokens": int, - "token_details": str, - "duration_ms": int, - "datetime_utc": str, - "response_json": str, - "error": str, - }, - pk="id", - foreign_keys=( - ("parent_message_hash", "messages", "hash"), - ("tip_message_hash", "messages", "hash"), - ("schema_id", "schemas", "id"), - ), - ) - - db["turn_tools"].create( - { - "turn_id": str, - "tool_id": int, - }, - pk=("turn_id", "tool_id"), - foreign_keys=( - ("turn_id", "turns", "id"), - ("tool_id", "tools", "id"), - ), - ) +# Literal text of one parts row: the text column when the part's text +# is pure literal, otherwise the literal segments of a text_ref payload +# - fragment content is deliberately not searchable. +TURN_SEARCH_LITERAL = """coalesce( + parts.text, + (select group_concat(json_extract(je.value, '$.literal'), '') + from json_each(parts.payload, '$.text_ref') je + where json_extract(je.value, '$.literal') is not null) +)""" - # A thread is a named, mutable pointer at a message - the only - # mutable thing in the new schema. Forking is a second pointer at an - # interior message. - db["threads"].create( - { - "id": str, - "name": str, - "tip_message_hash": str, - "forked_from": str, - "datetime_utc": str, - }, - pk="id", - foreign_keys=( - ("tip_message_hash", "messages", "hash"), - ("forked_from", "threads", "id"), - ), - ) +# Derives the searchable prompt and response text for turns. Serves both +# the migration backfill (turn_filter="") and the per-turn refresh in +# LogStore.log (turn_filter="and turns.id = :turn_id" - the slot appears +# in three places so the filtered form touches only that turn's chain). +TURN_SEARCH_INSERT_SQL = """ +with recursive output_messages(turn_id, hash) as ( + select turns.id, turns.tip_message_hash + from turns + where turns.tip_message_hash is not null + and (turns.parent_message_hash is null + or turns.tip_message_hash != turns.parent_message_hash) + {turn_filter} + union all + select om.turn_id, messages.parent_hash + from output_messages om + join messages on messages.hash = om.hash + join turns on turns.id = om.turn_id + where messages.parent_hash is not null + and (turns.parent_message_hash is null + or messages.parent_hash != turns.parent_message_hash) +), +prompt_text as ( + select turns.id as turn_id, + (select group_concat({LITERAL}, '') + from parts + where parts.message_hash = turns.parent_message_hash + and parts.type = 'text' + order by parts.position) as text + from turns + join messages on messages.hash = turns.parent_message_hash + where messages.role = 'user' {turn_filter} +), +response_text as ( + select om.turn_id, group_concat(part_text.text, '') as text + from output_messages om + join messages on messages.hash = om.hash and messages.role = 'assistant' + join ( + select parts.message_hash, parts.position, {LITERAL} as text + from parts where parts.type = 'text' + ) part_text on part_text.message_hash = om.hash + group by om.turn_id +) +insert into turn_search (turn_id, prompt, response) +select turns.id, + coalesce(prompt_text.text, ''), + coalesce(response_text.text, '') + from turns + left join prompt_text on prompt_text.turn_id = turns.id + left join response_text on response_text.turn_id = turns.id + where (coalesce(prompt_text.text, '') != '' + or coalesce(response_text.text, '') != '') {turn_filter} +""".replace("{LITERAL}", TURN_SEARCH_LITERAL) @migration -def m024_message_store_payloads(db): - # Reshape the m023 tables. Parts now carry the wire form of the part - # as a payload rather than a column per field, so a new part type or - # field needs no schema change and reading is Part.from_dict(). - # - # The payload stores large content by reference - fragment ids for - # text, attachment ids for binary - which is the whole point of the - # fragments feature: a novel is stored once and pointed at from every - # prompt about it. Hashing is unaffected either way, because identity - # is computed over the resolved content before anything is written. - # - # m023 shipped only in alphas and its tables are a mirror of data the - # legacy tables still hold in full, so this drops and recreates - # rather than carrying a data migration for a schema nobody has. - for table in ( - "turn_tools", - "turn_fragments", - "turns", - "threads", - "part_attachments", - "part_fragments", - "parts", - "messages", - ): - db[table].drop(ignore=True) - +def m023_message_store(db): + # The content-addressed message store, created in its final form. + # Nothing here is dropped, migrated or backfilled: these tables + # have never existed in any released version of LLM, so a database + # either gains them fresh or is in an unsupported development state + # - in which case the creates below fail loudly rather than + # touching whatever is there. db["messages"].create( { "hash": str, @@ -725,77 +640,11 @@ def m024_message_store_payloads(db): ) db["turn_fragments"].create_index(["fragment_id"]) - -# Literal text of one parts row: the text column when the part's text -# is pure literal, otherwise the literal segments of a text_ref payload -# - fragment content is deliberately not searchable. -TURN_SEARCH_LITERAL = """coalesce( - parts.text, - (select group_concat(json_extract(je.value, '$.literal'), '') - from json_each(parts.payload, '$.text_ref') je - where json_extract(je.value, '$.literal') is not null) -)""" - -# Derives the searchable prompt and response text for turns. Serves both -# the migration backfill (turn_filter="") and the per-turn refresh in -# LogStore.log (turn_filter="and turns.id = :turn_id" - the slot appears -# in three places so the filtered form touches only that turn's chain). -TURN_SEARCH_INSERT_SQL = """ -with recursive output_messages(turn_id, hash) as ( - select turns.id, turns.tip_message_hash - from turns - where turns.tip_message_hash is not null - and (turns.parent_message_hash is null - or turns.tip_message_hash != turns.parent_message_hash) - {turn_filter} - union all - select om.turn_id, messages.parent_hash - from output_messages om - join messages on messages.hash = om.hash - join turns on turns.id = om.turn_id - where messages.parent_hash is not null - and (turns.parent_message_hash is null - or messages.parent_hash != turns.parent_message_hash) -), -prompt_text as ( - select turns.id as turn_id, - (select group_concat({LITERAL}, '') - from parts - where parts.message_hash = turns.parent_message_hash - and parts.type = 'text' - order by parts.position) as text - from turns - join messages on messages.hash = turns.parent_message_hash - where messages.role = 'user' {turn_filter} -), -response_text as ( - select om.turn_id, group_concat(part_text.text, '') as text - from output_messages om - join messages on messages.hash = om.hash and messages.role = 'assistant' - join ( - select parts.message_hash, parts.position, {LITERAL} as text - from parts where parts.type = 'text' - ) part_text on part_text.message_hash = om.hash - group by om.turn_id -) -insert into turn_search (turn_id, prompt, response) -select turns.id, - coalesce(prompt_text.text, ''), - coalesce(response_text.text, '') - from turns - left join prompt_text on prompt_text.turn_id = turns.id - left join response_text on response_text.turn_id = turns.id - where (coalesce(prompt_text.text, '') != '' - or coalesce(response_text.text, '') != '') {turn_filter} -""".replace("{LITERAL}", TURN_SEARCH_LITERAL) - - -@migration -def m025_turn_search(db): # Searchable text per turn: the user's typed prompt (fragment - # content excluded) and the assistant's text output. An explicit id - # primary key because external-content FTS is keyed by rowid, and - # implicit rowids are not stable across VACUUM. + # content excluded) and the assistant's text output, kept fresh by + # LogStore.log. An explicit id primary key because external-content + # FTS is keyed by rowid, and implicit rowids are not stable across + # VACUUM. db["turn_search"].create( { "id": int, @@ -808,235 +657,23 @@ def m025_turn_search(db): ) db["turn_search"].create_index(["turn_id"], unique=True) db["turn_search"].enable_fts(["prompt", "response"], create_triggers=True) - # The backfill SQL reads parts.text, which m027 introduces - a - # database migrating from m024 straight through needs the column to - # exist before this runs. m027 skips the add when it is present. - if "text" not in db["parts"].columns_dict: - db["parts"].add_column("text", str) - db.execute(TURN_SEARCH_INSERT_SQL.format(turn_filter="")) - -@migration -def m027_parts_text_column(db): - # Literal text moves out of the JSON payload into its own column - - # raw, never escaped, never parsed - and the redundant "type" key - # (already a column) leaves every payload. What structure remains - # is stored as JSON, or NULL when the text column carries the whole - # part. Storage encoding only: hashes are computed over resolved - # message content before anything is written, so no hash changes. - if "text" not in db["parts"].columns_dict: - db["parts"].add_column("text", str) - # One transaction, and each row is only touched if it still carries - # the old keys - so an interrupted run can be retried without the - # already-migrated rows (whose payloads no longer have a text key) - # being blanked back to text=None. - with sqlite_transaction(db): - for row in list(db.query("select id, type, payload from parts")): - payload = json.loads(row["payload"]) if row["payload"] else {} - if "type" not in payload and "text" not in payload: - continue - payload.pop("type", None) - update: dict = {} - if row["type"] in ("text", "reasoning") and "text" in payload: - update["text"] = payload.pop("text") - update["payload"] = json.dumps(payload) if payload else None - db["parts"].update(row["id"], update) - - -@migration -def m028_tool_instantiations(db): - # Which configured toolbox instance served a tool call - e.g. that - # SQLite_query ran against SQLite("mydb.db"). Local execution + # Which configured toolbox instance served a tool call: the toolbox + # name, its plugin and its constructor arguments. Local execution # provenance, so it lives outside the hashed message tree, joined - # to the chain by tool_call_id (unique per call, stored on both the - # call and result parts). Deliberately the seed of a fuller + # to the chain by tool_call_id - and keyed by (turn_id, + # tool_call_id), because provider-supplied call ids are not + # guaranteed unique across turns. Deliberately the seed of a fuller # execution-events table: duration or exception details would be # additive columns here. db["tool_instantiations"].create( { + "turn_id": str, "tool_call_id": str, "name": str, "plugin": str, "arguments": str, }, - pk="tool_call_id", + pk=("turn_id", "tool_call_id"), + foreign_keys=(("turn_id", "turns", "id"),), ) - - -def _rehash_messages(db): - # Recompute every stored message hash from resolved content, - # bottom-up, and repoint everything that references one. Run by any - # migration that changes what participates in the hash; hashing - # from content also merges messages whose old hashes only ever - # differed by details the hash no longer covers. - if not db["messages"].exists() or not db["messages"].count: - return - # Runtime import - llm.logs imports this module at import time, but - # by the time a migration runs both modules are fully loaded. - from .logs import LogStore, message_hash - from .parts import Message - - store = LogStore.__new__(LogStore) # skip __init__, which migrates - store.db = db - - rows = {row["hash"]: row for row in db["messages"].rows} - parts_by_hash = store._load_parts(list(rows)) - - children: dict = {} - roots = [] - for row in rows.values(): - if row["parent_hash"] is None: - roots.append(row["hash"]) - else: - children.setdefault(row["parent_hash"], []).append(row["hash"]) - - mapping: dict = {} - queue = list(roots) - while queue: - old_hash = queue.pop() - row = rows[old_hash] - parent = row["parent_hash"] - message = Message( - role=row["role"], - parts=parts_by_hash.get(old_hash, []), - provider_metadata=_load_json(row["provider_metadata"]), - ) - mapping[old_hash] = message_hash(message, mapping.get(parent, parent)) - queue.extend(children.get(old_hash, [])) - - with sqlite_transaction(db): - # Primary keys are rewritten while other rows still reference - # them; defer enforcement to commit for connections that run - # with foreign keys enabled. The pragma only takes effect - # inside a transaction and resets itself at commit. - db.conn.execute("PRAGMA defer_foreign_keys = ON") - seen: set = set() - for old_hash, new_hash in mapping.items(): - if new_hash in seen: - # Two messages that differed only by attachment path - # are now one identity - keep the first, drop this - # one's rows and repoint its references below. - part_ids = [ - r["id"] - for r in db.query( - "select id from parts where message_hash = ?", [old_hash] - ) - ] - if part_ids: - placeholders = ",".join("?" * len(part_ids)) - db.execute( - f"delete from part_attachments where part_id in ({placeholders})", - part_ids, - ) - db.execute( - f"delete from part_fragments where part_id in ({placeholders})", - part_ids, - ) - db.execute( - f"delete from parts where id in ({placeholders})", part_ids - ) - db.execute("delete from messages where hash = ?", [old_hash]) - continue - seen.add(new_hash) - if new_hash != old_hash: - db.execute( - "update messages set hash = ? where hash = ?", [new_hash, old_hash] - ) - db.execute( - "update parts set message_hash = ? where message_hash = ?", - [new_hash, old_hash], - ) - for old_hash, new_hash in mapping.items(): - if new_hash == old_hash: - continue - db.execute( - "update messages set parent_hash = ? where parent_hash = ?", - [new_hash, old_hash], - ) - db.execute( - "update turns set parent_message_hash = ? " - "where parent_message_hash = ?", - [new_hash, old_hash], - ) - db.execute( - "update turns set tip_message_hash = ? where tip_message_hash = ?", - [new_hash, old_hash], - ) - db.execute( - "update threads set tip_message_hash = ? where tip_message_hash = ?", - [new_hash, old_hash], - ) - - -def _load_json(value): - return json.loads(value) if value else None - - -@migration -def m029_rehash_messages(db): - # Message hashes began identifying attachments by the sha256 of - # their content rather than the filesystem path they were loaded - # from. - _rehash_messages(db) - - -@migration -def m030_tool_instantiations_turn_scope(db): - # tool_call_id is not globally unique - providers with per-request - # counters can reuse the same id across independent turns - so the - # table is keyed by (turn_id, tool_call_id). Existing rows recover - # their turn through the stored parts; any row that cannot be - # matched is dropped rather than left able to collide. - if "turn_id" in db["tool_instantiations"].columns_dict: - return - db["tool_instantiations"].add_column("turn_id", str) - with sqlite_transaction(db): - # The tool result sits either in the turn's parent message, - # or - when a fresh user prompt followed the results - in the - # parent's own parent, one step up the same input segment. - db.execute(""" - update tool_instantiations set turn_id = ( - select turns.id from turns - join parts on parts.type = 'tool_result' - and json_extract(parts.payload, '$.tool_call_id') - = tool_instantiations.tool_call_id - where parts.message_hash = turns.parent_message_hash - or parts.message_hash = ( - select messages.parent_hash from messages - where messages.hash = turns.parent_message_hash - and messages.role = 'user' - ) - limit 1 - ) - """) - db.execute("delete from tool_instantiations where turn_id is null") - db["tool_instantiations"].transform(pk=("turn_id", "tool_call_id")) - - -@migration -def m031_turn_fragments_order_pk(db): - # The same fragment can be passed to a prompt more than once - m016 - # established that for the legacy tables - so order joins the key - # and repeats are preserved instead of collapsing to one row. - if "order" not in db["turn_fragments"].pks: - db["turn_fragments"].transform(pk=("turn_id", "fragment_id", "kind", "order")) - - -@migration -def m032_rehash_for_attachment_types(db): - # The canonical attachment form now includes the media type - the - # model sees it, so identical bytes sent as different types are - # different requests - and the content hash is recomputed from the - # actual bytes rather than a cached id. Both change hashes of - # attachment-bearing messages, so recompute the stored tree again. - _rehash_messages(db) - - -@migration -def m033_part_attachments_order_pk(db): - # A tool result can return the same attachment more than once; with - # order outside the key the second reference raised a UNIQUE error - # while the payload happily recorded both. Order joins the key, - # matching part_fragments and turn_fragments. - if "order" not in db["part_attachments"].pks: - db["part_attachments"].transform(pk=("part_id", "attachment_id", "order")) diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 7622fbfad..fc5d8c812 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -996,62 +996,18 @@ def test_tool_filters_match(self, store, mock_model): assert len(merged_log_rows(store, tool_names=["t"])) == 1 assert merged_log_rows(store, tool_names=["other"]) == [] - def test_m030_backfill_finds_results_behind_a_user_prompt(self, store, mock_model): - # A turn whose input ends [tool result, user prompt] keeps its - # provenance row through the m030 backfill - the result lives - # in the parent's parent, and a wrong-direction traversal used - # to delete these rows as unmatched. - class Notes(llm.Toolbox): - def __init__(self, path: str): - self.path = path - - mock_model.enqueue(["ok"]) - response = mock_model.prompt( - "next question", - messages=[llm.user("orig"), llm.assistant("first answer")], - tool_results=[ - llm.ToolResult( - name="Notes_read", - output="RESULT", - tool_call_id="c9", - instance=Notes("/tmp/n"), - ) - ], - ) - response.text() - response.log_to_db(store.db) - db = store.db - turn_id = next(iter(db["turns"].rows))["id"] - # Rewind the table to its pre-m030 shape - db["tool_instantiations"].transform(pk="tool_call_id", drop={"turn_id"}) - db.execute( - "delete from _llm_migrations" - " where name = 'm030_tool_instantiations_turn_scope'" - ) - migrate(db) - rows = list(db["tool_instantiations"].rows) - assert [row["turn_id"] for row in rows] == [turn_id] - -class TestMigrationRetrySafety: - def test_m027_retry_does_not_erase_migrated_text(self, store, mock_model): - # Simulate an interrupted m027: rows already in the new format - # but the migration not recorded as applied. Retrying must not - # blank the text column back out. - mock_model.enqueue(["Hello there"]) - response = mock_model.prompt("Hi") - response.text() - response.log_to_db(store.db) - before = {row["id"]: row["text"] for row in store.db["parts"].rows} - assert any(before.values()) - with store.db.conn: - store.db.execute( - "delete from _llm_migrations where name = 'm027_parts_text_column'" - ) - migrate(store.db) - after = {row["id"]: row["text"] for row in store.db["parts"].rows} - assert after == before - assert store.verify() == [] +class TestUnsupportedBranchDatabases: + def test_existing_message_store_tables_fail_loudly(self, tmp_path): + # The message store migration creates its tables in final form + # and assumes they do not exist - a database carrying tables + # from unreleased development revisions is an unsupported + # state, and the migration fails loudly rather than dropping + # or adapting whatever is there. + db = sqlite_utils.Database(str(tmp_path / "old-branch.db")) + db["messages"].create({"hash": str}, pk="hash") + with pytest.raises(sqlite3.OperationalError): + migrate(db) class TestAttachmentHashing: @@ -1204,132 +1160,6 @@ def test_deleting_the_file_is_detected_not_fatal(self, store, tmp_path): path.unlink() assert store.verify() == [tip] - def test_m032_reruns_the_rehash(self, store, tmp_path): - path = tmp_path / "x.png" - path.write_bytes(b"PNG BYTES") - messages = [ - Message( - role="user", - parts=[ - AttachmentPart( - attachment=Attachment(type="image/png", path=str(path)) - ) - ], - ), - llm.assistant("A fine image"), - ] - tip = store.ensure_chain(messages) - db = store.db - real = [row["hash"] for row in db.query("select hash from messages")] - fakes = {h: "b2:" + format(i, "032x") for i, h in enumerate(real)} - with db.conn: - for old, fake in fakes.items(): - db.execute("update messages set hash = ? where hash = ?", [fake, old]) - db.execute( - "update messages set parent_hash = ? where parent_hash = ?", - [fake, old], - ) - db.execute( - "update parts set message_hash = ? where message_hash = ?", - [fake, old], - ) - db.execute( - "delete from _llm_migrations where name = 'm032_rehash_for_attachment_types'" - ) - assert store.verify() != [] - migrate(db) - assert store.verify() == [] - assert store.load_chain(tip) == messages - - def test_m029_survives_foreign_key_enforcement(self, tmp_path): - # Rekeying messages.hash while parts and turns still reference - # it would fail immediately on a connection running with - # PRAGMA foreign_keys = ON - the migration defers enforcement - # to commit, inside a real transaction. - db = sqlite_utils.Database(str(tmp_path / "fk.db")) - db.conn.execute("PRAGMA foreign_keys = ON") - store = LogStore(db) - path = tmp_path / "x.png" - path.write_bytes(b"PNG BYTES") - messages = [ - Message( - role="user", - parts=[ - AttachmentPart( - attachment=Attachment(type="image/png", path=str(path)) - ) - ], - ), - llm.assistant("A fine image"), - ] - tip = store.ensure_chain(messages) - real = [row["hash"] for row in db.query("select hash from messages")] - fakes = {h: "b2:" + format(i, "032x") for i, h in enumerate(real)} - db.conn.execute("PRAGMA foreign_keys = OFF") - for old, fake in fakes.items(): - db.execute("update messages set hash = ? where hash = ?", [fake, old]) - db.execute( - "update messages set parent_hash = ? where parent_hash = ?", - [fake, old], - ) - db.execute( - "update parts set message_hash = ? where message_hash = ?", - [fake, old], - ) - db.execute("delete from _llm_migrations where name = 'm029_rehash_messages'") - db.conn.execute("PRAGMA foreign_keys = ON") - migrate(db) - assert store.verify() == [] - assert store.load_chain(tip) == messages - assert db.conn.execute("PRAGMA foreign_key_check").fetchall() == [] - - def test_m029_recomputes_stale_hashes(self, store, tmp_path): - # Build a real chain, then rewrite its hashes to bogus values - - # simulating rows written by the old path-based algorithm - and - # check the migration recomputes everything from content. - path = tmp_path / "x.png" - path.write_bytes(b"PNG BYTES") - messages = [ - Message( - role="user", - parts=[ - AttachmentPart( - attachment=Attachment(type="image/png", path=str(path)) - ) - ], - ), - llm.assistant("A fine image"), - ] - tip = store.ensure_chain(messages) - thread_id = store.create_thread(name="t", tip=tip) - db = store.db - real = [row["hash"] for row in db.query("select hash from messages")] - fakes = {h: "b2:" + format(i, "032x") for i, h in enumerate(real)} - with db.conn: - for old, fake in fakes.items(): - db.execute("update messages set hash = ? where hash = ?", [fake, old]) - db.execute( - "update messages set parent_hash = ? where parent_hash = ?", - [fake, old], - ) - db.execute( - "update parts set message_hash = ? where message_hash = ?", - [fake, old], - ) - db.execute( - "update threads set tip_message_hash = ? " - "where tip_message_hash = ?", - [fake, old], - ) - db.execute( - "delete from _llm_migrations where name = 'm029_rehash_messages'" - ) - assert store.verify() != [] - migrate(db) - assert store.verify() == [] - assert store.thread_tip(thread_id) == tip - assert store.load_chain(tip) == messages - class TestRepeatedAttachments: def test_a_tool_result_can_carry_the_same_attachment_twice(self, store): From 774e62bffd94ce649e3abee5d638a7768e2cdc92 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 19:59:35 +0000 Subject: [PATCH 168/258] Drop sqlite-utils 3.x support, require 4.0 or higher The 3.x branch of sqlite_transaction() could never work: sqlite-utils 3.x table helpers commit as they go, and any COMMIT inside the shim's savepoint block destroyed the savepoint out from under it - 23 tests failed on 3.x with "no such savepoint". sqlite-utils 4's Database.atomic() suspends per-call commits inside the block, which is the behavior the message store's write paths actually need, so the floor moves to 4.0 and the shim is deleted in favor of calling db.atomic() directly. Also retires the remaining placebo `with db.conn:` blocks - a no-op under 3.x autocommit, and under 4 the sqlite3 exit commit could commit a caller's open transaction. Embedding batch writes and Collection.delete() now use db.atomic() and are genuinely atomic; m007's wrapper around the rename is dropped rather than replaced since a single statement gains nothing from a transaction. CI loses the sqlite-utils version matrix axis and its install step - every job now tests the 4.x floor - with the cog check re-homed to the ubuntu/3.14 cell. The Justfile no longer needs --with to force a 4.x cog run. Verified: 942 tests, black, mypy, ruff and cog --check all pass on sqlite-utils 4.1.1, and the transaction-heavy suites pass on 4.0 exactly. Refs https://github.com/simonw/llm/pull/1562 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01538ebAg81aDhe7Qb5qwZcU --- .github/workflows/test.yml | 14 +---------- Justfile | 4 +-- docs/changelog.md | 1 + llm/cli.py | 3 +-- llm/embeddings.py | 4 +-- llm/embeddings_migrations.py | 2 +- llm/logs.py | 5 ++-- llm/migrations.py | 3 +-- llm/utils.py | 47 ------------------------------------ pyproject.toml | 2 +- 10 files changed, 12 insertions(+), 73 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 901330dd2..03ff77e92 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,14 +12,6 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - sqlite-utils-version: [""] - include: - - os: ubuntu-latest - python-version: "3.14" - sqlite-utils-version: "<4" - - os: ubuntu-latest - python-version: "3.14" - sqlite-utils-version: ">=4" steps: - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} @@ -31,15 +23,11 @@ jobs: - name: Install dependencies run: | pip install . --group dev - - name: Install sqlite-utils ${{ matrix.sqlite-utils-version }} - if: matrix.sqlite-utils-version != '' - run: | - pip install 'sqlite-utils${{ matrix.sqlite-utils-version }}' - name: Run tests run: | python -m pytest -vv - name: Check if cog needs to be run - if: matrix.sqlite-utils-version == '>=4' + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14' run: | cog --check \ -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" \ diff --git a/Justfile b/Justfile index 643c887fe..626ff09c4 100644 --- a/Justfile +++ b/Justfile @@ -11,7 +11,7 @@ echo " Black" uv run black . --check echo " cog" - uv run --with sqlite-utils==4.1.1 cog --check \ + uv run cog --check \ -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" \ README.md docs/*.md echo " mypy" @@ -25,7 +25,7 @@ # Rebuild docs with cog @cog: - uv run --with sqlite-utils==4.1.1 cog -r -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" docs/**/*.md docs/*.md README.md + uv run cog -r -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" docs/**/*.md docs/*.md README.md # Serve live docs on localhost:8000 @docs: cog diff --git a/docs/changelog.md b/docs/changelog.md index 70966d4e3..8561336cc 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,7 @@ ## Unreleased +- LLM now requires [sqlite-utils 4.0](https://sqlite-utils.datasette.io/en/stable/changelog.html) or higher. Database writes - logging a turn, storing fragments and their aliases, embedding batches, collection deletion - run inside real transactions using the `Database.atomic()` context manager introduced in sqlite-utils 4, so an interrupted write rolls back cleanly instead of leaving partial rows behind. [#1562](https://github.com/simonw/llm/pull/1562) - `llm logs` now reads both generations of log tables: history recorded by older versions of LLM in the legacy `responses` table is merged into the output alongside new turns, with rows from the dual-write era deduplicated by id. The `-m`, `-c`, `-f`, `-T`, `--tools` and `--schema` filters work across both. [#1562](https://github.com/simonw/llm/pull/1562) - Logging no longer writes the legacy tables: `Response.log_to_db()` records only the content-addressed message store. The legacy tables are kept in place as read-only history. Raw provider payloads (the old `prompt_json` and `response_json` columns) are no longer persisted - the stored message chain is the record of what was sent and returned. This means data that only ever lived in those raw payloads is no longer stored, notably the log probabilities returned by OpenAI completion models with `-o logprobs` - those remain available on the response object through the Python API, and rows logged by older versions of LLM keep the values they recorded. [#1562](https://github.com/simonw/llm/pull/1562) - `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. Results are most-relevant first, with prompt matches weighted well above response matches; `-l/--latest` switches to most-recent first. Searches also cover history recorded by older versions of LLM through its original `responses_fts` index. A longstanding bug where search results were ordered weakest match first is fixed. See {ref}`logging-search`. [#1562](https://github.com/simonw/llm/pull/1562) diff --git a/llm/cli.py b/llm/cli.py index 70731007a..f6798cb93 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -86,7 +86,6 @@ resolve_schema_input, schema_dsl, schema_summary, - sqlite_transaction, token_usage_string, truncate_string, ) @@ -2944,7 +2943,7 @@ def fragments_set(alias, fragment): on conflict(alias) do update set fragment_id = excluded.fragment_id; """ - with sqlite_transaction(db): + with db.atomic(): fragment_id = ensure_fragment(db, resolved) db.execute(alias_sql, {"alias": alias, "fragment_id": fragment_id}) diff --git a/llm/embeddings.py b/llm/embeddings.py index c044b9253..0afe0f8a9 100644 --- a/llm/embeddings.py +++ b/llm/embeddings.py @@ -214,7 +214,7 @@ def embed_multi_with_metadata( embeddings = list( self.model().embed_multi(item[1] for item in filtered_batch) ) - with self.db.conn: + with self.db.atomic(): cast(Table, self.db["embeddings"]).insert_all( ( { @@ -358,7 +358,7 @@ def delete(self): """ Delete the collection and its embeddings from the database """ - with self.db.conn: + with self.db.atomic(): self.db.execute("delete from embeddings where collection_id = ?", [self.id]) self.db.execute("delete from collections where id = ?", [self.id]) diff --git a/llm/embeddings_migrations.py b/llm/embeddings_migrations.py index 678aa4496..eab9428c3 100644 --- a/llm/embeddings_migrations.py +++ b/llm/embeddings_migrations.py @@ -63,7 +63,7 @@ def random_md5(): db.conn.create_function("temp_md5", 1, md5) db.conn.create_function("temp_random_md5", 0, random_md5) - with db.conn: + with db.atomic(): db.execute(""" update embeddings set content_hash = temp_md5(content) diff --git a/llm/logs.py b/llm/logs.py index c70b93960..44848a9a0 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -37,7 +37,6 @@ ensure_tool, make_schema_id, monotonic_ulid, - sqlite_transaction, ) __all__ = [ @@ -202,7 +201,7 @@ def _ensure_message( # Already stored - and because the hash covers the parent, # everything below it is stored too. return hash - with sqlite_transaction(self.db): + with self.db.atomic(): # Another writer can store the same hash between the check # above and this insert. Insert-or-ignore settles who won, # and only the winner writes the parts. @@ -455,7 +454,7 @@ def log(self, response, thread_id: str | None = None) -> str: timings, usage, which model answered - goes on the turn, because message rows are shared and so cannot carry provenance. """ - with sqlite_transaction(self.db): + with self.db.atomic(): return self._log_in_transaction(response, thread_id) def _log_in_transaction(self, response, thread_id: str | None) -> str: diff --git a/llm/migrations.py b/llm/migrations.py index bc6b44dfa..4fefa8c89 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -134,8 +134,7 @@ def m007_finish_logs_table(db): rename={"timestamp_utc": "datetime_utc"}, drop_foreign_keys=("chat_id",), ) - with db.conn: - db.execute("alter table log rename to logs") + db.execute("alter table log rename to logs") @migration diff --git a/llm/utils.py b/llm/utils.py index 1db9cf136..c4bfeaa0f 100644 --- a/llm/utils.py +++ b/llm/utils.py @@ -7,7 +7,6 @@ import textwrap import threading import time -from contextlib import contextmanager from typing import Any, Final import click @@ -733,49 +732,3 @@ def _fresh(ms: int) -> bytes: timestamp = int.to_bytes(ms, TIMESTAMP_LEN, "big") randomness = os.urandom(RANDOMNESS_LEN) return timestamp + randomness - - -@contextmanager -def sqlite_transaction(db): - """A real transaction over a sqlite_utils Database. - - ``with db.conn:`` is not one - sqlite-utils effectively runs in - autocommit, each helper call commits itself and the context manager - has nothing left to roll back. - - sqlite-utils 4 provides Database.atomic() with the semantics needed - here; the supported floor includes sqlite-utils 3.x, which does - not, so this mirrors it there. The raw connection is used on - purpose: routing BEGIN through db.execute trips sqlite-utils 4's - transaction bookkeeping, which commits it immediately. - """ - if hasattr(db, "atomic"): - with db.atomic(): - yield - return - conn = db.conn - if conn.in_transaction: - # Nested use: a savepoint, so an inner failure rolls back only - # the inner block, matching Database.atomic(). - savepoint = f"llm_txn_{os.urandom(8).hex()}" - conn.execute(f"SAVEPOINT {savepoint}") - try: - yield - except BaseException: - if conn.in_transaction: - conn.execute(f"ROLLBACK TO SAVEPOINT {savepoint}") - conn.execute(f"RELEASE SAVEPOINT {savepoint}") - raise - else: - conn.execute(f"RELEASE SAVEPOINT {savepoint}") - else: - conn.execute("BEGIN") - try: - yield - except BaseException: - if conn.in_transaction: - conn.execute("ROLLBACK") - raise - else: - if conn.in_transaction: - conn.execute("COMMIT") diff --git a/pyproject.toml b/pyproject.toml index 22c15252d..d661aa914 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "condense-json>=0.1.3", "openai>=2.32.0", "click-default-group>=1.2.3", - "sqlite-utils>=3.39.1", + "sqlite-utils>=4.0", "sqlite-migrate==0.1a2", "pydantic>=2.0.0", "PyYAML", From b0370035f89b8b1b6b75d0cd274008decd289efc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Jul 2026 13:13:30 -0700 Subject: [PATCH 169/258] Better sqlite-utils 4 changelog entry --- docs/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 8561336cc..46f68d1bf 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,9 +2,9 @@ ## Unreleased -- LLM now requires [sqlite-utils 4.0](https://sqlite-utils.datasette.io/en/stable/changelog.html) or higher. Database writes - logging a turn, storing fragments and their aliases, embedding batches, collection deletion - run inside real transactions using the `Database.atomic()` context manager introduced in sqlite-utils 4, so an interrupted write rolls back cleanly instead of leaving partial rows behind. [#1562](https://github.com/simonw/llm/pull/1562) - `llm logs` now reads both generations of log tables: history recorded by older versions of LLM in the legacy `responses` table is merged into the output alongside new turns, with rows from the dual-write era deduplicated by id. The `-m`, `-c`, `-f`, `-T`, `--tools` and `--schema` filters work across both. [#1562](https://github.com/simonw/llm/pull/1562) - Logging no longer writes the legacy tables: `Response.log_to_db()` records only the content-addressed message store. The legacy tables are kept in place as read-only history. Raw provider payloads (the old `prompt_json` and `response_json` columns) are no longer persisted - the stored message chain is the record of what was sent and returned. This means data that only ever lived in those raw payloads is no longer stored, notably the log probabilities returned by OpenAI completion models with `-o logprobs` - those remain available on the response object through the Python API, and rows logged by older versions of LLM keep the values they recorded. [#1562](https://github.com/simonw/llm/pull/1562) +- LLM now requires [sqlite-utils 4.0](https://sqlite-utils.datasette.io/en/stable/changelog.html#v4-0) or higher. Database writes - logging a turn, storing fragments and their aliases, embedding batches, collection deletion - run inside real transactions using the `Database.atomic()` context manager introduced in sqlite-utils 4, so an interrupted write rolls back cleanly instead of leaving partial rows behind. [#1562](https://github.com/simonw/llm/pull/1562) - `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. Results are most-relevant first, with prompt matches weighted well above response matches; `-l/--latest` switches to most-recent first. Searches also cover history recorded by older versions of LLM through its original `responses_fts` index. A longstanding bug where search results were ordered weakest match first is fixed. See {ref}`logging-search`. [#1562](https://github.com/simonw/llm/pull/1562) - The `parts` table stores literal text in a dedicated `text` column - raw and unescaped, so prose reads as prose in SQL queries - instead of wrapping it in JSON. The `payload` column now holds only structure (fragment references, tool call fields, provider metadata), without the redundant `type` key, and is NULL when the text column carries the whole part. Storage encoding only: message hashes are computed over resolved content and are unchanged. Existing databases are rewritten by a migration. [#1562](https://github.com/simonw/llm/pull/1562) - `llm logs` now shows which configured toolbox instance served each tool result - the toolbox name, plugin and constructor arguments, so a `SQLite_query` result records that it ran against `SQLite("mydb.db")`. Recorded in the new `tool_instantiations` table for new logs, and read from the legacy `tool_instances` data for history logged by older versions of LLM, which recorded it without ever displaying it. [#1562](https://github.com/simonw/llm/pull/1562) From dbff5c2fe1b7923cc90249094107a6ac02eb09f5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Jul 2026 13:50:01 -0700 Subject: [PATCH 170/258] Fixed an over-explanation comment --- llm/migrations.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/llm/migrations.py b/llm/migrations.py index 4fefa8c89..e40c0ac11 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -493,12 +493,7 @@ def m022_response_reasoning(db): @migration def m023_message_store(db): - # The content-addressed message store, created in its final form. - # Nothing here is dropped, migrated or backfilled: these tables - # have never existed in any released version of LLM, so a database - # either gains them fresh or is in an unsupported development state - # - in which case the creates below fail loudly rather than - # touching whatever is there. + # The content-addressed message store db["messages"].create( { "hash": str, From 516d44d1eeed419b4bddf08b40b9faa287ac3e9c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Jul 2026 13:51:51 -0700 Subject: [PATCH 171/258] Move the turn_search derivation SQL into llm/logs.py TURN_SEARCH_LITERAL and TURN_SEARCH_INSERT_SQL lived in migrations.py for the migration backfill, which no longer exists - the per-turn refresh in LogStore.log is their only consumer, so they move next to it. The turn_filter format slot goes too: with one caller left, the :turn_id filter is baked into the SQL directly. Co-Authored-By: Claude Fable 5 --- llm/logs.py | 70 +++++++++++++++++++++++++++++++++++++--- llm/migrations.py | 64 ------------------------------------ tests/test_logs_store.py | 4 +-- 3 files changed, 66 insertions(+), 72 deletions(-) diff --git a/llm/logs.py b/llm/logs.py index 44848a9a0..44dbd5074 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -21,7 +21,7 @@ import json from typing import Any -from .migrations import TURN_SEARCH_INSERT_SQL, migrate +from .migrations import migrate from .models import Attachment, _conversation_name from .parts import ( AttachmentPart, @@ -569,10 +569,7 @@ def _log_in_transaction(self, response, thread_id: str | None) -> str: # payloads, not from load_chain - resolving references would put # fragment content back into the searchable text. self.db["turn_search"].delete_where("turn_id = ?", [turn_id]) - self.db.execute( - TURN_SEARCH_INSERT_SQL.format(turn_filter="and turns.id = :turn_id"), - {"turn_id": turn_id}, - ) + self.db.execute(TURN_SEARCH_INSERT_SQL, {"turn_id": turn_id}) if thread_id is not None: self.db["threads"].update(thread_id, {"tip_message_hash": tip}) return turn_id @@ -796,6 +793,69 @@ def _now() -> str: order by {order_by}{limit} """ +# Literal text of one parts row: the text column when the part's text +# is pure literal, otherwise the literal segments of a text_ref payload +# - fragment content is deliberately not searchable. +TURN_SEARCH_LITERAL = """coalesce( + parts.text, + (select group_concat(json_extract(je.value, '$.literal'), '') + from json_each(parts.payload, '$.text_ref') je + where json_extract(je.value, '$.literal') is not null) +)""" + +# Derives the searchable prompt and response text for turns. Serves both +# the migration backfill (turn_filter="") and the per-turn refresh in +# LogStore.log (turn_filter="and turns.id = :turn_id" - the slot appears +# in three places so the filtered form touches only that turn's chain). +TURN_SEARCH_INSERT_SQL = """ +with recursive output_messages(turn_id, hash) as ( + select turns.id, turns.tip_message_hash + from turns + where turns.tip_message_hash is not null + and (turns.parent_message_hash is null + or turns.tip_message_hash != turns.parent_message_hash) + and turns.id = :turn_id + union all + select om.turn_id, messages.parent_hash + from output_messages om + join messages on messages.hash = om.hash + join turns on turns.id = om.turn_id + where messages.parent_hash is not null + and (turns.parent_message_hash is null + or messages.parent_hash != turns.parent_message_hash) +), +prompt_text as ( + select turns.id as turn_id, + (select group_concat({LITERAL}, '') + from parts + where parts.message_hash = turns.parent_message_hash + and parts.type = 'text' + order by parts.position) as text + from turns + join messages on messages.hash = turns.parent_message_hash + where messages.role = 'user' and turns.id = :turn_id +), +response_text as ( + select om.turn_id, group_concat(part_text.text, '') as text + from output_messages om + join messages on messages.hash = om.hash and messages.role = 'assistant' + join ( + select parts.message_hash, parts.position, {LITERAL} as text + from parts where parts.type = 'text' + ) part_text on part_text.message_hash = om.hash + group by om.turn_id +) +insert into turn_search (turn_id, prompt, response) +select turns.id, + coalesce(prompt_text.text, ''), + coalesce(response_text.text, '') + from turns + left join prompt_text on prompt_text.turn_id = turns.id + left join response_text on response_text.turn_id = turns.id + where (coalesce(prompt_text.text, '') != '' + or coalesce(response_text.text, '') != '') and turns.id = :turn_id +""".replace("{LITERAL}", TURN_SEARCH_LITERAL) + # Relevance ranking for -q. bm25 scores are negative-better, ascending # order is best-first. The prompt column is weighted well above the # response: what you typed is a stronger signal of what a turn is about diff --git a/llm/migrations.py b/llm/migrations.py index e40c0ac11..d538e3710 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -427,70 +427,6 @@ def m022_response_reasoning(db): db["responses"].add_column("reasoning", str) -# Literal text of one parts row: the text column when the part's text -# is pure literal, otherwise the literal segments of a text_ref payload -# - fragment content is deliberately not searchable. -TURN_SEARCH_LITERAL = """coalesce( - parts.text, - (select group_concat(json_extract(je.value, '$.literal'), '') - from json_each(parts.payload, '$.text_ref') je - where json_extract(je.value, '$.literal') is not null) -)""" - -# Derives the searchable prompt and response text for turns. Serves both -# the migration backfill (turn_filter="") and the per-turn refresh in -# LogStore.log (turn_filter="and turns.id = :turn_id" - the slot appears -# in three places so the filtered form touches only that turn's chain). -TURN_SEARCH_INSERT_SQL = """ -with recursive output_messages(turn_id, hash) as ( - select turns.id, turns.tip_message_hash - from turns - where turns.tip_message_hash is not null - and (turns.parent_message_hash is null - or turns.tip_message_hash != turns.parent_message_hash) - {turn_filter} - union all - select om.turn_id, messages.parent_hash - from output_messages om - join messages on messages.hash = om.hash - join turns on turns.id = om.turn_id - where messages.parent_hash is not null - and (turns.parent_message_hash is null - or messages.parent_hash != turns.parent_message_hash) -), -prompt_text as ( - select turns.id as turn_id, - (select group_concat({LITERAL}, '') - from parts - where parts.message_hash = turns.parent_message_hash - and parts.type = 'text' - order by parts.position) as text - from turns - join messages on messages.hash = turns.parent_message_hash - where messages.role = 'user' {turn_filter} -), -response_text as ( - select om.turn_id, group_concat(part_text.text, '') as text - from output_messages om - join messages on messages.hash = om.hash and messages.role = 'assistant' - join ( - select parts.message_hash, parts.position, {LITERAL} as text - from parts where parts.type = 'text' - ) part_text on part_text.message_hash = om.hash - group by om.turn_id -) -insert into turn_search (turn_id, prompt, response) -select turns.id, - coalesce(prompt_text.text, ''), - coalesce(response_text.text, '') - from turns - left join prompt_text on prompt_text.turn_id = turns.id - left join response_text on response_text.turn_id = turns.id - where (coalesce(prompt_text.text, '') != '' - or coalesce(response_text.text, '') != '') {turn_filter} -""".replace("{LITERAL}", TURN_SEARCH_LITERAL) - - @migration def m023_message_store(db): # The content-addressed message store diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index fc5d8c812..ce7ac4c6c 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -927,9 +927,7 @@ def test_failed_turn_write_rolls_back_the_whole_turn( response = mock_model.prompt("Hi") response.text() # Fail at the search refresh, one of the last steps of log() - monkeypatch.setattr( - "llm.logs.TURN_SEARCH_INSERT_SQL", "this is not sql {turn_filter}" - ) + monkeypatch.setattr("llm.logs.TURN_SEARCH_INSERT_SQL", "this is not sql") with pytest.raises(sqlite3.OperationalError): store.log(response) monkeypatch.undo() From ab46c8f7943b846b9615b99d72339ce372c66f47 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Jul 2026 14:51:10 -0700 Subject: [PATCH 172/258] Tool instance configs stored once, shown in the tools list too Real data showed the flat tool_instantiations design storing one configuration per CALL - four rows, two distinct configs - when the legacy tool_instances table had the right shape all along: one row per configured instance, referenced by id. Configurations now live in that shared table (ensure_tool_instance dedups by select-or-insert, across generations - the backfill matches rows the legacy writer recorded), tool_instantiations shrinks to (turn_id, tool_call_id, instance_id) link rows, and turn_tools gains instance_id so the tools list in llm logs shows which instance provided each tool at prompt time: - **Datasette_query**: `hash` - instance `Datasette({"url": ...})` previously visible only on tool results, after a call had run. The instance comes from the tool implementation's bound __self__ at logging time. m024 migrates in place - add columns, backfill instance references from the flat copies, drop the copies - rather than rebuilding, so databases created from recent commits of this branch upgrade without losing their rows. Co-Authored-By: Claude Fable 5 --- docs/changelog.md | 2 +- docs/logging.md | 9 +++-- llm/cli.py | 15 +++++++-- llm/logs.py | 72 ++++++++++++++++++++++++++++++++-------- llm/migrations.py | 27 +++++++++++++++ llm/utils.py | 18 ++++++++++ tests/test_logs_store.py | 58 ++++++++++++++++++++++---------- 7 files changed, 162 insertions(+), 39 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 46f68d1bf..0f201b061 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -7,7 +7,7 @@ - LLM now requires [sqlite-utils 4.0](https://sqlite-utils.datasette.io/en/stable/changelog.html#v4-0) or higher. Database writes - logging a turn, storing fragments and their aliases, embedding batches, collection deletion - run inside real transactions using the `Database.atomic()` context manager introduced in sqlite-utils 4, so an interrupted write rolls back cleanly instead of leaving partial rows behind. [#1562](https://github.com/simonw/llm/pull/1562) - `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. Results are most-relevant first, with prompt matches weighted well above response matches; `-l/--latest` switches to most-recent first. Searches also cover history recorded by older versions of LLM through its original `responses_fts` index. A longstanding bug where search results were ordered weakest match first is fixed. See {ref}`logging-search`. [#1562](https://github.com/simonw/llm/pull/1562) - The `parts` table stores literal text in a dedicated `text` column - raw and unescaped, so prose reads as prose in SQL queries - instead of wrapping it in JSON. The `payload` column now holds only structure (fragment references, tool call fields, provider metadata), without the redundant `type` key, and is NULL when the text column carries the whole part. Storage encoding only: message hashes are computed over resolved content and are unchanged. Existing databases are rewritten by a migration. [#1562](https://github.com/simonw/llm/pull/1562) -- `llm logs` now shows which configured toolbox instance served each tool result - the toolbox name, plugin and constructor arguments, so a `SQLite_query` result records that it ran against `SQLite("mydb.db")`. Recorded in the new `tool_instantiations` table for new logs, and read from the legacy `tool_instances` data for history logged by older versions of LLM, which recorded it without ever displaying it. [#1562](https://github.com/simonw/llm/pull/1562) +- `llm logs` now shows which configured toolbox instance provided each tool - in the tools list for a prompt and on each tool result, so a `SQLite_query` result records that it ran against `SQLite("mydb.db")`. Each distinct configuration is stored once in the `tool_instances` table and referenced from the new `tool_instantiations` and extended `turn_tools` tables; history logged by older versions of LLM shows the same details from the data it already recorded. [#1562](https://github.com/simonw/llm/pull/1562) - New documentation for the message store: the schema, the content-addressing hash contract, worked examples and a SQL cookbook, in {ref}`the logging documentation `. [#1562](https://github.com/simonw/llm/pull/1562) - Passing `prompt=`, `fragments=`, `attachments=` or `tool_results=` alongside `messages=` to `model.prompt()` or `conversation.prompt()` now appends that new input to the supplied message history, instead of silently omitting it from `prompt.messages` - previously the model could receive text that never appeared in the logged conversation. [#1562](https://github.com/simonw/llm/pull/1562) - A response logged outside of a conversation now gets a thread of its own, so responses logged through the Python API can be continued with `llm -c` - the same guarantee the `conversations` table used to provide. [#1562](https://github.com/simonw/llm/pull/1562) diff --git a/docs/logging.md b/docs/logging.md index 27379c5f4..974e9bc9d 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -541,10 +541,10 @@ The full schema for these tables appears in {ref}`the SQL schema section ` definitions were available to a turn, referencing the `tools` table. +- `turn_tools` - which {ref}`tool ` definitions were available to a turn, referencing the `tools` table. For toolbox-derived tools, `instance_id` references the `tool_instances` row recording which configured instance provided them - so the tools list in `llm logs` shows that `SQLite_query` came from `SQLite("mydb.db")` before any call has run. - `turn_fragments` - which fragments a turn was given, with their `kind` (`prompt` or `system`) and order. Provenance lives here rather than on the shared message rows, and this table is what powers `llm logs -f`. - `turn_search` - the searchable text of each turn: the literal prompt the user typed (fragment content excluded, via the `text_ref` literals described above) and the assistant's text output. An FTS5 index over this table, `turn_search_fts`, is what powers {ref}`llm logs -q `. Derived from the stored parts when a turn is logged; a turn with no prompt or response text, such as a pure tool call, gets no row. -- `tool_instantiations` - which configured {ref}`toolbox ` instance served a tool call: the toolbox name, its plugin and its constructor arguments, keyed by `(turn_id, tool_call_id)` - call ids supplied by providers are not guaranteed unique across turns. Message rows are shared between conversations and so cannot carry this kind of local execution provenance; this table joins to the chain from outside it, and is what lets `llm logs` show that a `SQLite_query` call ran against `SQLite("mydb.db")`. +- `tool_instantiations` - which configured {ref}`toolbox ` instance served a tool call, as a reference into the shared `tool_instances` table (each distinct configuration is stored once), keyed by `(turn_id, tool_call_id)` - call ids supplied by providers are not guaranteed unique across turns. Message rows are shared between conversations and so cannot carry this kind of local execution provenance; this table joins to the chain from outside it. (logging-message-store-queries)= @@ -857,6 +857,7 @@ CREATE TABLE "turns" ( CREATE TABLE "turn_tools" ( "turn_id" TEXT REFERENCES "turns"("id"), "tool_id" INTEGER REFERENCES "tools"("id"), + "instance_id" INTEGER REFERENCES "tool_instances"("id"), PRIMARY KEY ("turn_id", "tool_id") ); @@ -884,9 +885,7 @@ CREATE VIRTUAL TABLE "turn_search_fts" USING FTS5 ( CREATE TABLE "tool_instantiations" ( "turn_id" TEXT REFERENCES "turns"("id"), "tool_call_id" TEXT, - "name" TEXT, - "plugin" TEXT, - "arguments" TEXT, + "instance_id" INTEGER REFERENCES "tool_instances"("id"), PRIMARY KEY ("turn_id", "tool_call_id") ); diff --git a/llm/cli.py b/llm/cli.py index f6798cb93..29982df93 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -2115,16 +2115,27 @@ def _display_fragments(fragments, title): if row["tools"]: click.echo("\n### Tools\n") for tool in row["tools"]: + instance = tool.get("instance") + instance_bit = "" + if instance: + arguments = instance["arguments"] + instance_bit = " - instance `{}({})`".format( + instance["name"], + arguments if arguments and arguments != "{}" else "", + ) if tool["hash"] in seen_tool_hashes: click.echo( - "- **{}**: `{}`".format(tool["name"], tool["hash"][:7]) + "- **{}**: `{}`{}".format( + tool["name"], tool["hash"][:7], instance_bit + ) ) else: seen_tool_hashes.add(tool["hash"]) click.echo( - "- **{}**: `{}`
\n{}
\n Arguments: `{}`".format( + "- **{}**: `{}`{}
\n{}
\n Arguments: `{}`".format( tool["name"], tool["hash"], + instance_bit, textwrap.indent( (tool["description"] or "").rstrip(), " " ), diff --git a/llm/logs.py b/llm/logs.py index 44dbd5074..0e3f6e973 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -35,6 +35,7 @@ from .utils import ( ensure_fragment, ensure_tool, + ensure_tool_instance, make_schema_id, monotonic_ulid, ) @@ -515,8 +516,26 @@ def _log_in_transaction(self, response, thread_id: str | None) -> str: replace=True, ) for tool in response.prompt.tools: + # A toolbox-derived tool's implementation is a method bound + # to the configured instance - record which one, as a + # reference into the shared tool_instances table. + instance = getattr(tool.implementation, "__self__", None) + config = getattr(instance, "_config", None) self.db["turn_tools"].insert( - {"turn_id": turn_id, "tool_id": ensure_tool(self.db, tool)}, + { + "turn_id": turn_id, + "tool_id": ensure_tool(self.db, tool), + "instance_id": ( + ensure_tool_instance( + self.db, + tool.name.split("_")[0], + tool.plugin, + json.dumps(config), + ) + if config is not None + else None + ), + }, replace=True, ) # Which fragments this call was given - provenance, so it belongs @@ -550,16 +569,19 @@ def _log_in_transaction(self, response, thread_id: str | None) -> str: { "turn_id": turn_id, "tool_call_id": tool_result.tool_call_id, - "name": tool_result.name.split("_")[0], - "plugin": next( - ( - tool.plugin - for tool in response.prompt.tools - if tool.name == tool_result.name + "instance_id": ensure_tool_instance( + self.db, + tool_result.name.split("_")[0], + next( + ( + tool.plugin + for tool in response.prompt.tools + if tool.name == tool_result.name + ), + None, ), - None, + json.dumps(config), ), - "arguments": json.dumps(config), }, replace=True, ) @@ -1109,12 +1131,29 @@ def log_row_extras(store: "LogStore", row: dict) -> dict: # input_schema is rendered as a dict, so decode it here rather than # handing the caller the raw JSON text out of the column. tools = [ - {**tool_row, "input_schema": json.loads(tool_row["input_schema"] or "{}")} + { + "id": tool_row["id"], + "hash": tool_row["hash"], + "name": tool_row["name"], + "description": tool_row["description"], + "input_schema": json.loads(tool_row["input_schema"] or "{}"), + "instance": ( + { + "name": tool_row["instance_name"], + "arguments": tool_row["instance_arguments"], + } + if tool_row["instance_name"] + else None + ), + } for tool_row in store.db.query( """ select tools.id, tools.hash, tools.name, tools.description, - tools.input_schema + tools.input_schema, tool_instances.name as instance_name, + tool_instances.arguments as instance_arguments from tools join turn_tools on turn_tools.tool_id = tools.id + left join tool_instances + on tool_instances.id = turn_tools.instance_id where turn_tools.turn_id = ? """, [row["id"]], @@ -1235,9 +1274,13 @@ def _instances_by_tool_call_id( } for row in store.db.query( f""" - select tool_call_id, name, plugin, arguments + select tool_instantiations.tool_call_id, tool_instances.name, + tool_instances.plugin, tool_instances.arguments from tool_instantiations - where turn_id = ? and tool_call_id in ({placeholders}) + join tool_instances + on tool_instances.id = tool_instantiations.instance_id + where tool_instantiations.turn_id = ? + and tool_instantiations.tool_call_id in ({placeholders}) """, [turn_id] + tool_call_ids, ) @@ -1461,7 +1504,8 @@ def merged_log_rows( 'hash', t.hash, 'name', t.name, 'description', t.description, - 'input_schema', json(t.input_schema) + 'input_schema', json(t.input_schema), + 'instance', null )) from tools t join tool_responses tr on t.id = tr.tool_id diff --git a/llm/migrations.py b/llm/migrations.py index d538e3710..2a9f445de 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -607,3 +607,30 @@ def m023_message_store(db): pk=("turn_id", "tool_call_id"), foreign_keys=(("turn_id", "turns", "id"),), ) + + +@migration +def m024_tool_instance_references(db): + # Tool instance configurations - e.g. Datasette("https://...") - + # are stored once in the shared tool_instances table and referenced + # by id, instead of being copied onto every row that mentions them: + # tool_instantiations gains instance_id in place of its + # name/plugin/arguments copies, and turn_tools gains instance_id so + # the tools list can show which configured instance provided each + # tool. + from .utils import ensure_tool_instance + + db["turn_tools"].add_column("instance_id", int, fk="tool_instances", fk_col="id") + db["tool_instantiations"].add_column( + "instance_id", int, fk="tool_instances", fk_col="id" + ) + for row in list(db["tool_instantiations"].rows): + db["tool_instantiations"].update( + (row["turn_id"], row["tool_call_id"]), + { + "instance_id": ensure_tool_instance( + db, row["name"], row["plugin"], row["arguments"] + ) + }, + ) + db["tool_instantiations"].transform(drop={"name", "plugin", "arguments"}) diff --git a/llm/utils.py b/llm/utils.py index c4bfeaa0f..4bf48d50d 100644 --- a/llm/utils.py +++ b/llm/utils.py @@ -513,6 +513,24 @@ def ensure_tool(db, tool): ).fetchone()[0] +def ensure_tool_instance(db, name, plugin, arguments) -> int: + """Row id in tool_instances for this configuration, storing each + distinct (plugin, name, arguments) once however many turns and + calls it serves.""" + match = db.execute( + "select id from tool_instances where name is ? " + "and plugin is ? and arguments is ?", + [name, plugin, arguments], + ).fetchone() + if match: + return match[0] + return ( + db["tool_instances"] + .insert({"name": name, "plugin": plugin, "arguments": arguments}) + .last_pk + ) + + def maybe_fenced_code(content: str) -> str: "Return the content as a fenced code block if it looks like code" is_code = False diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index ce7ac4c6c..1cb5a0521 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -800,15 +800,41 @@ def __init__(self, path: str): response.text() response.log_to_db(store.db) turn_id = next(iter(store.db["turns"].rows))["id"] - assert list(store.db["tool_instantiations"].rows) == [ - { - "tool_call_id": "tc_1", - "name": "Notes", - "plugin": None, - "arguments": '{"path": "/tmp/notes"}', - "turn_id": turn_id, - } - ] + link = next(iter(store.db["tool_instantiations"].rows)) + assert link["turn_id"] == turn_id + assert link["tool_call_id"] == "tc_1" + instance = store.db["tool_instances"].get(link["instance_id"]) + assert instance["name"] == "Notes" + assert instance["plugin"] is None + assert instance["arguments"] == '{"path": "/tmp/notes"}' + + def test_turn_tools_reference_the_configured_instance(self, store, mock_model): + class Notes(llm.Toolbox): + def __init__(self, path: str): + self.path = path + + def read(self) -> str: + "Read the notes" + return "hi" + + for prompt in ("first", "second"): + mock_model.enqueue(["ok"]) + response = mock_model.prompt(prompt, tools=[Notes("/tmp/x")]) + response.text() + response.log_to_db(store.db) + + # One instance row however many turns it serves + assert store.db["tool_instances"].count == 1 + instance_ids = {row["instance_id"] for row in store.db["turn_tools"].rows} + assert len(instance_ids) == 1 + + # And the tools list in the display carries it + rows = merged_log_rows(store) + tools = log_row_extras(store, rows[0])["tools"] + assert tools[0]["instance"] == { + "name": "Notes", + "arguments": '{"path": "/tmp/x"}', + } def test_tool_instantiations_are_scoped_by_turn(self, store, mock_model): # Providers with per-request counters can reuse a tool_call_id @@ -832,14 +858,12 @@ def __init__(self, path: str): ) response.text() response.log_to_db(store.db) - arguments = { - row["turn_id"]: row["arguments"] - for row in store.db["tool_instantiations"].rows - } - assert sorted(arguments.values()) == [ - '{"path": "/tmp/one"}', - '{"path": "/tmp/two"}', - ] + arguments = sorted(row["arguments"] for row in store.db.query(""" + select tool_instances.arguments from tool_instantiations + join tool_instances + on tool_instances.id = tool_instantiations.instance_id + """)) + assert arguments == ['{"path": "/tmp/one"}', '{"path": "/tmp/two"}'] def test_successive_library_turns_extend_the_thread(self, store, mock_model): conversation = mock_model.conversation() From 4786a7ab8b2ec9d9fc1219c277596eb9bb75aeba Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Jul 2026 14:59:12 -0700 Subject: [PATCH 173/258] Backfill turn_tools.instance_id for rows that predate the column Turns logged before instance_id existed show nothing in the tools list - m024 had no source to fill them, since the flat design never recorded a tools-list instance at all. m025 fills the NULLs from the instance that served calls in the same thread, matched by toolbox name prefix. Co-Authored-By: Claude Fable 5 --- llm/migrations.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/llm/migrations.py b/llm/migrations.py index 2a9f445de..a29b1f140 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -634,3 +634,27 @@ def m024_tool_instance_references(db): }, ) db["tool_instantiations"].transform(drop={"name", "plugin", "arguments"}) + + +@migration +def m025_turn_tools_instance_backfill(db): + # turn_tools rows written before instance_id existed have NULL + # there, so the tools list shows nothing for them. The instance + # that served calls in the same thread, matched by toolbox name + # prefix, is the right value for these development-era rows. + with db.atomic(): + db.execute(""" + update turn_tools set instance_id = ( + select ti.instance_id from tool_instantiations ti + join turns turn_a on turn_a.id = ti.turn_id + join turns turn_b on turn_b.id = turn_tools.turn_id + and turn_b.thread_id = turn_a.thread_id + join tool_instances instance + on instance.id = ti.instance_id + join tools on tools.id = turn_tools.tool_id + where tools.name = instance.name + or tools.name like instance.name || '\\_%' escape '\\' + limit 1 + ) + where instance_id is null + """) From 773039bc3f00842b118e6c02292be638953d852e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Jul 2026 16:10:50 -0700 Subject: [PATCH 174/258] Group the tools list by toolbox instance Two tools from one configured instance repeated the instance details on every line. The instance now renders once, with its tools nested beneath it; plain function tools stay flat: ### Tools - `Datasette({"url": "https://..."})`: - **Datasette_query**: `1aa6d35...`
Execute provided SQLite SQL query... - **Datasette_schema**: `24ec615...`
... Co-Authored-By: Claude Fable 5 --- llm/cli.py | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index 29982df93..694cca181 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -2114,34 +2114,46 @@ def _display_fragments(fragments, title): # Show tool calls and results if row["tools"]: click.echo("\n### Tools\n") - for tool in row["tools"]: - instance = tool.get("instance") - instance_bit = "" - if instance: - arguments = instance["arguments"] - instance_bit = " - instance `{}({})`".format( - instance["name"], - arguments if arguments and arguments != "{}" else "", - ) + + def echo_tool(tool, indent=""): if tool["hash"] in seen_tool_hashes: - click.echo( - "- **{}**: `{}`{}".format( - tool["name"], tool["hash"][:7], instance_bit - ) - ) + block = "- **{}**: `{}`".format(tool["name"], tool["hash"][:7]) else: seen_tool_hashes.add(tool["hash"]) - click.echo( - "- **{}**: `{}`{}
\n{}
\n Arguments: `{}`".format( + block = ( + "- **{}**: `{}`
\n{}
\n Arguments: `{}`".format( tool["name"], tool["hash"], - instance_bit, textwrap.indent( (tool["description"] or "").rstrip(), " " ), json.dumps(tool["input_schema"]["properties"]), ) ) + click.echo(textwrap.indent(block, indent)) + + # Tools provided by the same configured toolbox instance + # nest beneath one instance line rather than repeating it + plain_tools = [] + by_instance: dict = {} + for tool in row["tools"]: + instance = tool.get("instance") + if instance: + key = (instance["name"], instance["arguments"]) + by_instance.setdefault(key, []).append(tool) + else: + plain_tools.append(tool) + for tool in plain_tools: + echo_tool(tool) + for (name, arguments), instance_tools in by_instance.items(): + click.echo( + "- `{}({})`:".format( + name, + arguments if arguments and arguments != "{}" else "", + ) + ) + for tool in instance_tools: + echo_tool(tool, " ") if row["tool_results"]: click.echo("\n### Tool results\n") for tool_result in row["tool_results"]: From 2d090a243cdee7a8cfa5b82a02ebfc1e36b2b69a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Jul 2026 16:27:18 -0700 Subject: [PATCH 175/258] Improved display of toolbox instances in llm logs --- llm/cli.py | 31 ++++++++++--------------------- tests/test_llm_logs.py | 4 ++-- tests/test_tools.py | 4 ++-- 3 files changed, 14 insertions(+), 25 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index 694cca181..9d2fb4acb 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -2120,15 +2120,13 @@ def echo_tool(tool, indent=""): block = "- **{}**: `{}`".format(tool["name"], tool["hash"][:7]) else: seen_tool_hashes.add(tool["hash"]) - block = ( - "- **{}**: `{}`
\n{}
\n Arguments: `{}`".format( - tool["name"], - tool["hash"], - textwrap.indent( - (tool["description"] or "").rstrip(), " " - ), - json.dumps(tool["input_schema"]["properties"]), - ) + block = "- **{}**: `{}` \n{} \n Arguments: `{}`".format( + tool["name"], + tool["hash"], + textwrap.indent( + (tool["description"] or "").rstrip(), " " + ), + json.dumps(tool["input_schema"]["properties"]), ) click.echo(textwrap.indent(block, indent)) @@ -2169,22 +2167,13 @@ def echo_tool(tool, indent=""): elif attachment.get("content"): desc += f"<{attachment['content_length']:,} bytes>" attachments += f"\n - {desc}" - instance = tool_result.get("instance") - instance_bit = "" - if instance: - arguments = instance["arguments"] - instance_bit = " - instance `{}({})`".format( - instance["name"], - arguments if arguments and arguments != "{}" else "", - ) click.echo( - "- **{}**: `{}`{}
\n{}{}{}".format( + "- **{}**: `{}` \n{}{}{}".format( tool_result["name"], tool_result["tool_call_id"], - instance_bit, _fenced_block(tool_result["output"]), ( - "
\n **Error**: {}\n".format( + " \n **Error**: {}\n".format( tool_result["exception"] ) if tool_result["exception"] @@ -2232,7 +2221,7 @@ def echo_tool(tool, indent=""): click.echo("### Tool calls\n") for tool_call in row["tool_calls"]: click.echo( - "- **{}**: `{}`
\n{}".format( + "- **{}**: `{}` \n{}".format( tool_call["name"], tool_call["tool_call_id"], _format_tool_call_arguments(tool_call["arguments"]), diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index 11104dfda..51446f123 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -1100,7 +1100,7 @@ def demo(): assert ( "### Tool results\n" "\n" - "- **demo**: `tc_TCID`
\n" + "- **demo**: `tc_TCID` \n" " ```\n" " one\n" " two\n" @@ -1176,7 +1176,7 @@ def demo(timeout: int, options: list): assert ( "### Tool calls\n" "\n" - "- **demo**: `tc_TCID`
\n" + "- **demo**: `tc_TCID` \n" " timeout: `120`\n" ' options: ``["`tick`"]``\n' ) in normalized_output diff --git a/tests/test_tools.py b/tests/test_tools.py index afae2cf1c..c1214e34d 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -656,10 +656,10 @@ def test_tool_errors(async_): assert log_text_result.exit_code == 0 normalized_log_text = re.sub(r"tc_[0-9a-z]{26}", "tc_TCID", log_text_result.output) assert ( - "- **trigger_error**: `tc_TCID`
\n" + "- **trigger_error**: `tc_TCID` \n" " ```\n" " Error: Error!\n" - " ```
\n" + " ``` \n" " **Error**: Exception: Error!\n" ) in normalized_log_text From db90a0a4a707c1ca1349d585f6407df295b1c161 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Jul 2026 17:17:32 -0700 Subject: [PATCH 176/258] message_tree view renders conversation trees as indented text The m026 migration creates a message_tree view over the message store: one row per message, depth-first with forks as siblings, carrying the tree's root_hash for filtering, the timestamp of the earliest turn that recorded each message, and the names of any tools executed. Documented in the logging page with the SQL behind a details reveal. Co-Authored-By: Claude Fable 5 --- docs/logging.md | 105 +++++++++++++++++++++++---------------- llm/migrations.py | 62 +++++++++++++++++++++++ tests/test_logs_store.py | 87 ++++++++++++++++++++++++++++++++ 3 files changed, 210 insertions(+), 44 deletions(-) diff --git a/docs/logging.md b/docs/logging.md index 974e9bc9d..338986ac5 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -550,56 +550,73 @@ The full schema for these tables appears in {ref}`the SQL schema section The SQL query behind the message_tree view ```sql -with recursive -siblings as ( - select - hash, - parent_hash, - count(*) over (partition by parent_hash) as branches - from messages +with recursive msg as ( + select m.hash, m.parent_hash, m.role, m.rowid as rid, + replace(coalesce( + nullif(p.text, ''), + (select f.content from part_fragments pf + join fragments f on f.id = pf.fragment_id + where pf.part_id = p.id + order by pf."order" limit 1), + '[' || coalesce(p.type, 'empty') || ']' + ), char(10), ' ') as text, + (select group_concat(p2.tool_name, ', ') from parts p2 + where p2.message_hash = m.hash and p2.type = 'tool_result' + and p2.tool_name is not null) as tools + from messages m + left join parts p on p.message_hash = m.hash and p.position = 0 ), -walk as ( - select - messages.hash, - 0 as indent, - printf('%08d', messages.rowid) as sort_key - from messages - where messages.parent_hash is null +tree as ( + select hash, text, tools, 0 as depth, + printf('%012d', rid) as path, hash as root_hash + from msg where parent_hash is null union all - select - siblings.hash, - walk.indent + (siblings.branches > 1), - walk.sort_key || '/' || printf('%08d', messages.rowid) - from siblings - join messages on messages.hash = siblings.hash - join walk on siblings.parent_hash = walk.hash + select msg.hash, msg.text, msg.tools, t.depth + 1, + t.path || '/' || printf('%012d', msg.rid), + t.root_hash + from msg join tree t on msg.parent_hash = t.hash +), +turn_chain as ( + select t.id as turn_id, t.datetime_utc, m.hash, m.parent_hash + from turns t join messages m on m.hash = t.tip_message_hash + union all + select tc.turn_id, tc.datetime_utc, m.hash, m.parent_hash + from turn_chain tc join messages m on m.hash = tc.parent_hash ) select - substr(' ', 1, walk.indent * 4) - || messages.role || ': ' - || coalesce( - parts.text, - ( - select group_concat( - coalesce( - json_extract(piece.value, '$.literal'), - (select content from fragments - where id = json_extract(piece.value, '$.fragment')) - ), - '' order by piece.key - ) - from json_each(json_extract(parts.payload, '$.text_ref')) as piece - ), - parts.type - ) as entry -from walk -join messages on messages.hash = walk.hash -left join parts on parts.message_hash = messages.hash -order by walk.sort_key, parts.position; -``` + t.root_hash, + strftime('%Y-%m-%d %H:%M:%S', + (select min(tc.datetime_utc) from turn_chain tc where tc.hash = t.hash) + ) as datetime, + replace(hex(zeroblob(t.depth)), '00', ' ') || substr(t.text, 1, 60) + as message, + coalesce(t.tools, '') as tools, + t.hash as message_hash, + t.path +from tree t +order by t.path +``` + +
Every turn that was given a specific fragment, via the `turn_fragments` provenance table - set `:fragment_hash` to a hash from `llm fragments`: diff --git a/llm/migrations.py b/llm/migrations.py index a29b1f140..73d282c0d 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -658,3 +658,65 @@ def m025_turn_tools_instance_backfill(db): ) where instance_id is null """) + + +MESSAGE_TREE_SQL = """ +with recursive msg as ( + select m.hash, m.parent_hash, m.role, m.rowid as rid, + replace(coalesce( + nullif(p.text, ''), + (select f.content from part_fragments pf + join fragments f on f.id = pf.fragment_id + where pf.part_id = p.id + order by pf."order" limit 1), + '[' || coalesce(p.type, 'empty') || ']' + ), char(10), ' ') as text, + (select group_concat(p2.tool_name, ', ') from parts p2 + where p2.message_hash = m.hash and p2.type = 'tool_result' + and p2.tool_name is not null) as tools + from messages m + left join parts p on p.message_hash = m.hash and p.position = 0 +), +tree as ( + select hash, text, tools, 0 as depth, + printf('%012d', rid) as path, hash as root_hash + from msg where parent_hash is null + union all + select msg.hash, msg.text, msg.tools, t.depth + 1, + t.path || '/' || printf('%012d', msg.rid), + t.root_hash + from msg join tree t on msg.parent_hash = t.hash +), +turn_chain as ( + select t.id as turn_id, t.datetime_utc, m.hash, m.parent_hash + from turns t join messages m on m.hash = t.tip_message_hash + union all + select tc.turn_id, tc.datetime_utc, m.hash, m.parent_hash + from turn_chain tc join messages m on m.hash = tc.parent_hash +) +select + t.root_hash, + strftime('%Y-%m-%d %H:%M:%S', + (select min(tc.datetime_utc) from turn_chain tc where tc.hash = t.hash) + ) as datetime, + replace(hex(zeroblob(t.depth)), '00', ' ') || substr(t.text, 1, 60) + as message, + coalesce(t.tools, '') as tools, + t.hash as message_hash, + t.path +from tree t +order by t.path +""".strip() + + +@migration +def m026_message_tree_view(db): + # A readable rendering of the message store: every conversation + # tree as indented text, one row per message, depth-first with + # siblings in insertion order. root_hash identifies a tree - filter + # on it to isolate one conversation and its forks. datetime is the + # earliest turn that recorded the message, since shared message + # rows carry no timestamp of their own. Kept ordered by including + # path in the output - selecting from the view preserves tree order + # only while sorted by path. + db.create_view("message_tree", MESSAGE_TREE_SQL) diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 1cb5a0521..047352840 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -1614,3 +1614,90 @@ def test_hashing_still_ignores_key_order(self, store): ) assert one == two assert store.db["messages"].count == 1 + + +# ---- the message_tree view ------------------------------------------- + + +class TestMessageTreeView: + def rows(self, store): + return list(store.db.query("select * from message_tree order by path")) + + def test_forks_render_as_indented_siblings(self, store): + thread = store.create_thread() + fork_point = store.append( + thread, [llm.user("Question"), llm.assistant("Answer")] + ) + store.append(thread, [llm.user("Follow-up A")]) + forked = store.fork(fork_point) + store.append(forked, [llm.user("Follow-up B")]) + + assert [row["message"] for row in self.rows(store)] == [ + "Question", + " Answer", + " Follow-up A", + " Follow-up B", + ] + + def test_every_message_carries_its_tree_root_hash(self, store): + store.append(store.create_thread(), [llm.user("One"), llm.assistant("1")]) + store.append(store.create_thread(), [llm.user("Two")]) + + rows = self.rows(store) + assert [row["message"] for row in rows] == ["One", " 1", "Two"] + one, reply, two = rows + # A root is its own root; descendants inherit it; separate + # conversations get separate roots. + assert one["root_hash"] == one["message_hash"] + assert reply["root_hash"] == one["message_hash"] + assert two["root_hash"] == two["message_hash"] + + def test_tool_results_show_their_tool_names(self, store): + store.append( + store.create_thread(), + [ + llm.user("Search"), + Message( + role="tool", + parts=[ + ToolResultPart(name="lookup", output="42", tool_call_id="c1"), + ToolResultPart(name="fetch", output="x", tool_call_id="c2"), + ], + ), + ], + ) + prompt, tool = self.rows(store) + assert prompt["tools"] == "" + assert tool["message"].strip() == "[tool_result]" + assert sorted(tool["tools"].split(", ")) == ["fetch", "lookup"] + + def test_fragment_referenced_text_is_resolved(self, store): + novel = "Call me Ishmael, but keep this fragment out of the parts table." + store.ensure_chain([llm.user(f"{novel}\nwho?")], fragments=[novel]) + (row,) = self.rows(store) + # The stored part holds a text_ref, not the text itself; the view + # displays the fragment's content in its place. + assert row["message"] == novel[:60] + + def test_text_is_flattened_and_truncated(self, store): + store.append(store.create_thread(), [llm.user("line one\nline two " + "x" * 100)]) + (row,) = self.rows(store) + assert row["message"] == ("line one line two " + "x" * 100)[:60] + + def test_datetime_comes_from_the_turn_that_logged_the_message( + self, store, mock_model + ): + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + store.log(response) + rows = self.rows(store) + # Both messages were recorded by the same turn, so they share + # its timestamp. + assert len({row["datetime"] for row in rows}) == 1 + assert rows[0]["datetime"] is not None + + def test_messages_no_turn_has_recorded_have_no_datetime(self, store): + store.append(store.create_thread(), [llm.user("Hi")]) + (row,) = self.rows(store) + assert row["datetime"] is None From aa2e58fe02695f5c7597749ec07cb216fbb8c23c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Jul 2026 17:16:07 -0700 Subject: [PATCH 177/258] llm -c now reconstructs configured toolbox instances Continuing a conversation that used a toolbox previously failed with 'Tool(s) Datasette_query not found' because loaded_tools contained the method-level tool names. Now load_conversation() collapses tools that reference a tool_instances row into a single -T style spec string built from the recorded name and arguments, which _gather_tools() already knows how to instantiate. llm chat -c gets the same fix through the shared _get_conversation_tools() helper. Closes #1092 Co-Authored-By: Claude Fable 5 --- docs/changelog.md | 1 + docs/usage.md | 4 +++- llm/cli.py | 51 +++++++++++++++++++++++++++++-------------- llm/models.py | 8 +++---- tests/test_plugins.py | 47 ++++++++++++++++++++++++++++++++++----- 5 files changed, 84 insertions(+), 27 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 0f201b061..32433e511 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -8,6 +8,7 @@ - `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. Results are most-relevant first, with prompt matches weighted well above response matches; `-l/--latest` switches to most-recent first. Searches also cover history recorded by older versions of LLM through its original `responses_fts` index. A longstanding bug where search results were ordered weakest match first is fixed. See {ref}`logging-search`. [#1562](https://github.com/simonw/llm/pull/1562) - The `parts` table stores literal text in a dedicated `text` column - raw and unescaped, so prose reads as prose in SQL queries - instead of wrapping it in JSON. The `payload` column now holds only structure (fragment references, tool call fields, provider metadata), without the redundant `type` key, and is NULL when the text column carries the whole part. Storage encoding only: message hashes are computed over resolved content and are unchanged. Existing databases are rewritten by a migration. [#1562](https://github.com/simonw/llm/pull/1562) - `llm logs` now shows which configured toolbox instance provided each tool - in the tools list for a prompt and on each tool result, so a `SQLite_query` result records that it ran against `SQLite("mydb.db")`. Each distinct configuration is stored once in the `tool_instances` table and referenced from the new `tool_instantiations` and extended `turn_tools` tables; history logged by older versions of LLM shows the same details from the data it already recorded. [#1562](https://github.com/simonw/llm/pull/1562) +- Continuing a conversation that used a configured toolbox now works: `llm -c` and `llm chat -c` reconstruct each toolbox instance from its recorded configuration, so a conversation started with `-T 'Datasette("https://datasette.io/content")'` can be continued without repeating the `-T` option. Previously this failed with a "Tool not found" error. Toolbox instances are rebuilt fresh, so any in-memory state from the earlier prompt is not carried over. [#1562](https://github.com/simonw/llm/pull/1562) - New documentation for the message store: the schema, the content-addressing hash contract, worked examples and a SQL cookbook, in {ref}`the logging documentation `. [#1562](https://github.com/simonw/llm/pull/1562) - Passing `prompt=`, `fragments=`, `attachments=` or `tool_results=` alongside `messages=` to `model.prompt()` or `conversation.prompt()` now appends that new input to the supplied message history, instead of silently omitting it from `prompt.messages` - previously the model could receive text that never appeared in the logged conversation. [#1562](https://github.com/simonw/llm/pull/1562) - A response logged outside of a conversation now gets a thread of its own, so responses logged through the Python API can be continued with `llm -c` - the same guarantee the `conversations` table used to provide. [#1562](https://github.com/simonw/llm/pull/1562) diff --git a/docs/usage.md b/docs/usage.md index 6295f343e..6f6e82e4f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -208,7 +208,9 @@ Toolboxes always start with a capital letter. They can be configured by passing - Single JSON value: `ToolboxName("hello")` or `ToolboxName([1,2,3])` - Key-value pairs: `ToolboxName(name="test", count=5, items=[1,2])` - treated the same as `{"name": "test", "count": 5, "items": [1, 2]}`, all values must be valid JSON -Toolboxes are not currently supported with the `llm -c` option, but they work well with `llm chat`. Try chatting with the Datasette content database like this: +Continuing a conversation with `llm -c` or `llm chat -c` reconstructs any toolboxes from the configuration recorded in the logs, so you don't need to repeat the `-T` option. Each instance is rebuilt fresh, which means any in-memory state a toolbox accumulated during the earlier prompt does not carry over. + +Toolboxes work well with `llm chat`. Try chatting with the Datasette content database like this: ```bash llm chat -T 'Datasette("https://datasette.io/content")' --td diff --git a/llm/cli.py b/llm/cli.py index 9d2fb4acb..18f2746ec 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -1423,21 +1423,39 @@ def load_conversation( pass # Plugin tools recorded against the first turn, for the same - # reuse-on-continue behaviour the rebuilt responses provide. - conversation.loaded_tools = [ - tool_row["name"] - for tool_row in db.query( - """ - select tools.name from tools - join turn_tools on turn_tools.tool_id = tools.id - where tools.plugin is not null - and turn_tools.turn_id = ( - select id from turns where thread_id = ? order by id limit 1 - ) - """, - [conversation_id], + # reuse-on-continue behaviour the rebuilt responses provide. Tools + # that came from a toolbox are collapsed into a single spec string + # like Datasette({"url": "..."}) - the same format -T accepts - so + # the instance can be reconstructed with its configuration. + loaded_tools = [] + seen_instance_ids = set() + for tool_row in db.query( + """ + select tools.name, turn_tools.instance_id, + tool_instances.name as instance_name, + tool_instances.arguments as instance_arguments + from tools + join turn_tools on turn_tools.tool_id = tools.id + left join tool_instances on tool_instances.id = turn_tools.instance_id + where tools.plugin is not null + and turn_tools.turn_id = ( + select id from turns where thread_id = ? order by id limit 1 ) - ] + """, + [conversation_id], + ): + if tool_row["instance_id"] is None: + loaded_tools.append(tool_row["name"]) + elif tool_row["instance_id"] not in seen_instance_ids: + seen_instance_ids.add(tool_row["instance_id"]) + arguments = tool_row["instance_arguments"] + if arguments and arguments != "{}": + loaded_tools.append( + "{}({})".format(tool_row["instance_name"], arguments) + ) + else: + loaded_tools.append(tool_row["instance_name"]) + conversation.loaded_tools = loaded_tools return conversation @@ -4112,6 +4130,7 @@ def _get_conversation_tools(conversation, tools): # Only tools from plugins: return [tool.name for tool in initial_tools if tool.plugin] elif conversation.loaded_tools: - # Conversation loaded from the message store - the tool names - # were read from turn_tools instead of rebuilt responses. + # Conversation loaded from the message store - tool names and + # toolbox specs were read from turn_tools instead of rebuilt + # responses. return list(conversation.loaded_tools) diff --git a/llm/models.py b/llm/models.py index 1278255e4..1c149127b 100644 --- a/llm/models.py +++ b/llm/models.py @@ -632,10 +632,10 @@ class _BaseConversation: # exact message list, so reasoning signatures and provider metadata # survive being reloaded. loaded_messages: list[Any] | None = None - # Names of plugin tools recorded against this conversation's first - # turn in storage. Read when the conversation was loaded from the - # message store, where there are no rebuilt responses to copy - # prompt.tools from. + # Plugin tool names and toolbox specs (e.g. 'Datasette({"url": ...})') + # recorded against this conversation's first turn in storage. Read + # when the conversation was loaded from the message store, where + # there are no rebuilt responses to copy prompt.tools from. loaded_tools: list[str] | None = None @classmethod diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 671c802fc..1d8fb5f2b 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -789,16 +789,30 @@ def after_call(tool, tool_call, tool_result): } ] - # Should show an error if you attempt to llm -c with configured toolboxes + # The stored instance configuration comes back as a -T style spec + conversation = cli.load_conversation(None) + assert conversation.loaded_tools == [ + "Filesystem({})".format(json.dumps({"path": str(my_dir2)})) + ] + + # llm -c should reconstruct the configured toolbox from the log result5 = runner.invoke( cli.cli, - ["-c", "list them again"], + ["-c", json.dumps({"tool_calls": [{"name": "Filesystem_list_files"}]})], ) - assert result5.exit_code == 1 - assert ( - "Error: Tool(s) Filesystem_list_files not found. Available tools:" - in result5.output + assert result5.exit_code == 0 + tool_results = json.loads( + "[" + + result5.output.rsplit('"tool_results": [', 1)[1].rsplit("]", 1)[0] + + "]" ) + assert tool_results == [ + { + "name": "Filesystem_list_files", + "output": json.dumps([str(other_path)]), + "tool_call_id": ANY, + } + ] # Test the logging worked rows = tool_activity_rows(logs_db) @@ -858,6 +872,27 @@ def after_call(tool, tool_call, tool_result): } ], }, + # The llm -c continuation, using the reconstructed toolbox + { + "model": "echo", + "tool_calls": [{"name": "Filesystem_list_files", "arguments": "{}"}], + "tool_results": [], + }, + { + "model": "echo", + "tool_calls": [], + "tool_results": [ + { + "name": "Filesystem_list_files", + "output": json.dumps([str(other_path)]), + "instance": { + "name": "Filesystem", + "plugin": "ToolboxPlugin", + "arguments": json.dumps({"path": str(my_dir2)}), + }, + } + ], + }, ] finally: From 771650307bd6eb51245cc10a2e48f813286521b1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Jul 2026 17:24:01 -0700 Subject: [PATCH 178/258] Black --- tests/test_logs_store.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 047352840..e68812424 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -1680,7 +1680,9 @@ def test_fragment_referenced_text_is_resolved(self, store): assert row["message"] == novel[:60] def test_text_is_flattened_and_truncated(self, store): - store.append(store.create_thread(), [llm.user("line one\nline two " + "x" * 100)]) + store.append( + store.create_thread(), [llm.user("line one\nline two " + "x" * 100)] + ) (row,) = self.rows(store) assert row["message"] == ("line one line two " + "x" * 100)[:60] From b627576f54e0a114247c684740f75c32fe0df93d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 08:27:36 -0700 Subject: [PATCH 179/258] Release 0.32rc1 Refs #1478, #1553, #1562, #1563, #1566 --- docs/changelog.md | 37 ++++++++++++++++++-------- docs/plugins/advanced-model-plugins.md | 1 + pyproject.toml | 2 +- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 32433e511..41f7c169f 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,18 +1,33 @@ # Changelog -## Unreleased +## 0.32rc1 +This release candidate for 0.32 introduces a new database schema for logging prompts and responses that captures full details of the interaction with the underlying LLM, and de-duplicates those records using a **content-addressed message store**. + +Upgrading to this RC will create those new tables and start logging to them. Existing data in the `responses` table will be left unaffected, and the `logs` command will read from both old and new tables. New interactions will only be written to the new tables. + +### Changes to SQLite logging + +- See {ref}`the message store documentation ` for details of the new logging schema. +- `llm prompt` logs are now written to a new set of tables, not the legacy tables. +- New documented {ref}`Response.log_to_db() ` Python API for writing to a SQLite logs database. - `llm logs` now reads both generations of log tables: history recorded by older versions of LLM in the legacy `responses` table is merged into the output alongside new turns, with rows from the dual-write era deduplicated by id. The `-m`, `-c`, `-f`, `-T`, `--tools` and `--schema` filters work across both. [#1562](https://github.com/simonw/llm/pull/1562) -- Logging no longer writes the legacy tables: `Response.log_to_db()` records only the content-addressed message store. The legacy tables are kept in place as read-only history. Raw provider payloads (the old `prompt_json` and `response_json` columns) are no longer persisted - the stored message chain is the record of what was sent and returned. This means data that only ever lived in those raw payloads is no longer stored, notably the log probabilities returned by OpenAI completion models with `-o logprobs` - those remain available on the response object through the Python API, and rows logged by older versions of LLM keep the values they recorded. [#1562](https://github.com/simonw/llm/pull/1562) -- LLM now requires [sqlite-utils 4.0](https://sqlite-utils.datasette.io/en/stable/changelog.html#v4-0) or higher. Database writes - logging a turn, storing fragments and their aliases, embedding batches, collection deletion - run inside real transactions using the `Database.atomic()` context manager introduced in sqlite-utils 4, so an interrupted write rolls back cleanly instead of leaving partial rows behind. [#1562](https://github.com/simonw/llm/pull/1562) -- `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. Results are most-relevant first, with prompt matches weighted well above response matches; `-l/--latest` switches to most-recent first. Searches also cover history recorded by older versions of LLM through its original `responses_fts` index. A longstanding bug where search results were ordered weakest match first is fixed. See {ref}`logging-search`. [#1562](https://github.com/simonw/llm/pull/1562) -- The `parts` table stores literal text in a dedicated `text` column - raw and unescaped, so prose reads as prose in SQL queries - instead of wrapping it in JSON. The `payload` column now holds only structure (fragment references, tool call fields, provider metadata), without the redundant `type` key, and is NULL when the text column carries the whole part. Storage encoding only: message hashes are computed over resolved content and are unchanged. Existing databases are rewritten by a migration. [#1562](https://github.com/simonw/llm/pull/1562) -- `llm logs` now shows which configured toolbox instance provided each tool - in the tools list for a prompt and on each tool result, so a `SQLite_query` result records that it ran against `SQLite("mydb.db")`. Each distinct configuration is stored once in the `tool_instances` table and referenced from the new `tool_instantiations` and extended `turn_tools` tables; history logged by older versions of LLM shows the same details from the data it already recorded. [#1562](https://github.com/simonw/llm/pull/1562) -- Continuing a conversation that used a configured toolbox now works: `llm -c` and `llm chat -c` reconstruct each toolbox instance from its recorded configuration, so a conversation started with `-T 'Datasette("https://datasette.io/content")'` can be continued without repeating the `-T` option. Previously this failed with a "Tool not found" error. Toolbox instances are rebuilt fresh, so any in-memory state from the earlier prompt is not carried over. [#1562](https://github.com/simonw/llm/pull/1562) -- New documentation for the message store: the schema, the content-addressing hash contract, worked examples and a SQL cookbook, in {ref}`the logging documentation `. [#1562](https://github.com/simonw/llm/pull/1562) -- Passing `prompt=`, `fragments=`, `attachments=` or `tool_results=` alongside `messages=` to `model.prompt()` or `conversation.prompt()` now appends that new input to the supplied message history, instead of silently omitting it from `prompt.messages` - previously the model could receive text that never appeared in the logged conversation. [#1562](https://github.com/simonw/llm/pull/1562) -- A response logged outside of a conversation now gets a thread of its own, so responses logged through the Python API can be continued with `llm -c` - the same guarantee the `conversations` table used to provide. [#1562](https://github.com/simonw/llm/pull/1562) -- Chain responses now yield a single space at the boundary between rounds when neither side supplies its own whitespace, so streamed output no longer runs the end of one response into the start of the next. The separator is synthesized at the chain level for display only - previously plugins worked around this by emitting a real space event, which was recorded as a whitespace-only part in the log. [#1562](https://github.com/simonw/llm/pull/1562) +- Raw provider payloads (the old `prompt_json` and `response_json` columns) are no longer persisted - the stored message chain is the record of what was sent and returned. This means data that only ever lived in those raw payloads is no longer stored, notably the log probabilities returned by OpenAI completion models with `-o logprobs`. +- `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. See {ref}`logging-search`. +- `llm logs` now shows which configured toolbox instance provided each tool - in the tools list for a prompt and on each tool result, so a `SQLite_query` result records that it ran against `SQLite("mydb.db")`. +- New {ref}`message_tree SQL view ` that renders each conversation thread as an indented text outline, for exploring the message store directly with SQL. [#1562](https://github.com/simonw/llm/pull/1562) + +### Other changes + +- New OpenAI models: `gpt-5.6-sol`, `gpt-5.6-terra` and `gpt-5.6-luna`. +- Removed OpenAI models that are no longer available via the OpenAI API: `chatgpt-4o-latest`, the `gpt-4o` and `gpt-4o-mini` audio preview models, `gpt-4-32k`, `gpt-4-1106-preview`, `gpt-4-0125-preview`, the `gpt-4.5` preview models, `o1-preview`, `o1-mini` and `gpt-5.1-chat-latest`. [#1553](https://github.com/simonw/llm/issues/1553) +- Continuing a conversation that used a configured toolbox now works: `llm -c` and `llm chat -c` reconstruct each toolbox instance from its recorded configuration, so a conversation started with `-T 'Datasette("https://datasette.io/content")'` can be continued without repeating the `-T` option. +- New `llm prompt --json` option which outputs a JSON array describing the prompt and the response, in the same format as `llm logs --json`. This works even with `--no-log` or logging turned off, in which case the response is logged to a temporary in-memory database to build the JSON. [#1566](https://github.com/simonw/llm/pull/1566) +- New `response.execute_tool_call(tool_call)` method (awaitable on async responses) that executes a single tool call and returns its `ToolResult`. This is designed for plugins wrapping provider SDKs that orchestrate the tool loop themselves, described in {ref}`the advanced model plugins documentation `. +- Documentation for the `responses: true` setting in `extra-openai-models.yaml`, which opts a custom OpenAI model into the Responses API instead of Chat Completions. See {ref}`openai-extra-models`. +- Fixed a bug where the system prompt was omitted from the pre-computed `prompt.messages` on the first turn of a conversation, so models that build their request from that message list never received it. Thanks, [Niall Smart](https://github.com/niallsmart). [#1478](https://github.com/simonw/llm/issues/1478) +- Fixed a bug on Windows where a fragment argument holding an absolute path such as `C:\Users\demo\notes.txt` was mistaken for a reference to a `c:` fragment loader plugin. Fragment arguments that match an existing file path now always resolve as files. [#1563](https://github.com/simonw/llm/issues/1563) +- LLM now requires [sqlite-utils 4.0](https://sqlite-utils.datasette.io/en/stable/changelog.html#v4-0) or higher. (v0_31_1)= ## 0.31.1 (2026-07-09) diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index 40de87137..4bf814d3a 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -380,6 +380,7 @@ response.add_tool_call( ) ``` +(advanced-model-plugins-execute-tool-call)= ### Provider-managed local tool calls Some provider SDKs orchestrate the tool loop themselves: they call a local callback with generated arguments, wait for that callback to return a result, and then continue the same model response. Plugins for those providers can delegate the actual invocation to LLM using `response.execute_tool_call()`: diff --git a/pyproject.toml b/pyproject.toml index d661aa914..18cf6b572 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm" -version = "0.32a3" +version = "0.32rc1" description = "CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine." readme = { file = "README.md", content-type = "text/markdown" } authors = [ From b957bf768bf2386599d61728bd0cd047660d4469 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 08:33:34 -0700 Subject: [PATCH 180/258] Updated version in docs with cog --- docs/fragments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/fragments.md b/docs/fragments.md index ac29b7235..5734a333e 100644 --- a/docs/fragments.md +++ b/docs/fragments.md @@ -30,7 +30,7 @@ from importlib.metadata import version llm_version = version("llm") cog.out(f'The URL will be fetched with the user-agent `llm/{llm_version} (https://llm.datasette.io/)`.') ]]]--> -The URL will be fetched with the user-agent `llm/0.32a3 (https://llm.datasette.io/)`. +The URL will be fetched with the user-agent `llm/0.32rc1 (https://llm.datasette.io/)`. The `-f` option can be used multiple times to combine together multiple fragments. From 618ae73d5bd7b53cbd3631bce2193d8c9c3110f6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 08:40:47 -0700 Subject: [PATCH 181/258] Stable #fragment for new 0.32rc1 release Fragment was #rc1 prior to this change --- docs/changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.md b/docs/changelog.md index 41f7c169f..6096e6f66 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,6 @@ # Changelog +(v0_32_rc1)= ## 0.32rc1 This release candidate for 0.32 introduces a new database schema for logging prompts and responses that captures full details of the interaction with the underlying LLM, and de-duplicates those records using a **content-addressed message store**. From c88e9102615e0b821fe1be9707453c63f6eb8aa2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 09:13:15 -0700 Subject: [PATCH 182/258] llm logs backup note in 0.32rc1 changelog --- docs/changelog.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 6096e6f66..f2391a139 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -7,6 +7,10 @@ This release candidate for 0.32 introduces a new database schema for logging pro Upgrading to this RC will create those new tables and start logging to them. Existing data in the `responses` table will be left unaffected, and the `logs` command will read from both old and new tables. New interactions will only be written to the new tables. +You can create a backup of your logs database prior to upgrading using: + + llm logs backup logs-backup.db + ### Changes to SQLite logging - See {ref}`the message store documentation ` for details of the new logging schema. From c2838a49d539a7a0dcb6309825ae23ace5a42ee5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 12:19:44 -0700 Subject: [PATCH 183/258] Switch default model to GPT-5.6 Luna, closes #1576 --- README.md | 6 +-- docs/aliases.md | 10 ++--- docs/changelog.md | 4 ++ docs/help.md | 14 +++---- docs/index.md | 6 +-- docs/openai-models.md | 4 +- docs/python-api.md | 66 +++++++++++++++---------------- docs/schemas.md | 4 +- docs/setup.md | 4 +- docs/templates.md | 4 +- docs/usage.md | 60 ++++++++++++++--------------- llm/__init__.py | 2 +- llm/cli.py | 14 +++---- tests/conftest.py | 50 ++++++++++++++++++++++++ tests/test_fragments_cli.py | 11 +++++- tests/test_keys.py | 20 ++++++++-- tests/test_llm.py | 77 +++++++++++++++---------------------- tests/test_templates.py | 14 +++---- 18 files changed, 212 insertions(+), 158 deletions(-) diff --git a/README.md b/README.md index 6c6952b70..55ee84726 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ If you have an [OpenAI API key](https://platform.openai.com/api-keys) key you ca # Paste your OpenAI API key into this llm keys set openai -# Run a prompt (with the default gpt-4o-mini model) +# Run a prompt (with the default gpt-5.6-luna model) llm "Ten fun names for a pet pelican" # Extract text from an image @@ -91,12 +91,12 @@ Run prompts against [Gemini](https://aistudio.google.com/apikey) or [Anthropic]( llm install llm-gemini llm keys set gemini # Paste Gemini API key here -llm -m gemini-2.0-flash 'Tell me fun facts about Mountain View' +llm -m gemini-3.5-flash 'Tell me fun facts about Mountain View' llm install llm-anthropic llm keys set anthropic # Paste Anthropic API key here -llm -m claude-4-opus 'Impress me with wild facts about turnips' +llm -m claude-5-opus 'Impress me with wild facts about turnips' ``` You can also [install a plugin](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#installing-plugins) to access models that can run on your local device. If you use [Ollama](https://ollama.com/): diff --git a/docs/aliases.md b/docs/aliases.md index 33e7e33dd..a685f4d33 100644 --- a/docs/aliases.md +++ b/docs/aliases.md @@ -66,15 +66,15 @@ Example output: The `llm aliases set ` command can be used to add a new alias: ```bash -llm aliases set mini gpt-4o-mini +llm aliases set luna gpt-5.6-luna ``` You can also pass one or more `-q search` options to set an alias on the first model matching those search terms: ```bash -llm aliases set mini -q 4o -q mini +llm aliases set luna -q gpt -q luna ``` -Now you can run the `gpt-4o-mini` model using the `mini` alias like this: +Now you can run the `gpt-5.6-luna` model using the `luna` alias like this: ```bash -llm -m mini 'An epic Greek-style saga about a cheesecake that builds a SQL database from scratch' +llm -m luna 'An epic Greek-style saga about a cheesecake that builds a SQL database from scratch' ``` Aliases can be set for both regular models and {ref}`embedding models ` using the same command. To set an alias of `oai` for the OpenAI `ada-002` embedding model use this: ```bash @@ -110,4 +110,4 @@ To view the content of that file, run this: ```bash cat "$(llm aliases path)" -``` \ No newline at end of file +``` diff --git a/docs/changelog.md b/docs/changelog.md index f2391a139..14a0df99e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- The default model for users who have not set their own default is now [GPT-5.6 Luna](https://developers.openai.com/api/docs/models/gpt-5.6-luna). It was previously [GPT-4o mini](https://developers.openai.com/api/docs/models/gpt-4o-mini). Luna is a much better and more recent model, albeit slightly more expensive - $0.20 per million input tokens and $1.20 per million output tokens, compared to $0.15/$0.60 for 4o mini. You can switch back to 4o mini using `llm models default gpt-4o-mini`, or switch to [GPT-5 nano](https://developers.openai.com/api/docs/models/gpt-5-nano), an even cheaper default model ($0.05/$0.40), using `llm models default gpt-5-nano`. + (v0_32_rc1)= ## 0.32rc1 diff --git a/docs/help.md b/docs/help.md index d84f6616f..df697511d 100644 --- a/docs/help.md +++ b/docs/help.md @@ -101,7 +101,7 @@ Usage: llm prompt [OPTIONS] [PROMPT] Examples: llm 'Capital of France?' - llm 'Capital of France?' -m gpt-4o + llm 'Capital of France?' -m gpt-5.5 llm 'Capital of France?' -s 'answer in Spanish' Multi-modal models can be called with attachments like this: @@ -459,7 +459,7 @@ Usage: llm models options show [OPTIONS] MODEL Example usage: - llm models options show gpt-4o + llm models options show gpt-4.1 Options: -h, --help Show this message and exit. @@ -474,7 +474,7 @@ Usage: llm models options set [OPTIONS] MODEL KEY VALUE Example usage: - llm models options set gpt-4o temperature 0.5 + llm models options set gpt-4.1 temperature 0.5 Options: -h, --help Show this message and exit. @@ -489,9 +489,9 @@ Usage: llm models options clear [OPTIONS] MODEL [KEY] Example usage: - llm models options clear gpt-4o + llm models options clear gpt-4.1 # Or for a single option - llm models options clear gpt-4o temperature + llm models options clear gpt-4.1 temperature Options: -h, --help Show this message and exit. @@ -694,12 +694,12 @@ Usage: llm aliases set [OPTIONS] ALIAS [MODEL_ID] Example usage: - llm aliases set mini gpt-4o-mini + llm aliases set luna gpt-5.6-luna Alternatively you can omit the model ID and specify one or more -q options. The first model matching all of those query strings will be used. - llm aliases set mini -q 4o -q mini + llm aliases set luna -q gpt -q luna Options: -q, --query TEXT Set alias for model matching these strings diff --git a/docs/index.md b/docs/index.md index 4536f5d1a..8cfe293f0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -44,7 +44,7 @@ If you have an [OpenAI API key](https://platform.openai.com/api-keys) key you ca # Paste your OpenAI API key into this llm keys set openai -# Run a prompt (with the default gpt-4o-mini model) +# Run a prompt (with the default gpt-5.6-luna model) llm "Ten fun names for a pet pelican" # Extract text from an image @@ -58,12 +58,12 @@ Run prompts against [Gemini](https://aistudio.google.com/apikey) or [Anthropic]( llm install llm-gemini llm keys set gemini # Paste Gemini API key here -llm -m gemini-2.0-flash 'Tell me fun facts about Mountain View' +llm -m gemini-3.5-flash 'Tell me fun facts about Mountain View' llm install llm-anthropic llm keys set anthropic # Paste Anthropic API key here -llm -m claude-4-opus 'Impress me with wild facts about turnips' +llm -m claude-5-opus 'Impress me with wild facts about turnips' ``` You can also {ref}`install a plugin ` to access models that can run on your local device. If you use [Ollama](https://ollama.com/): ```bash diff --git a/docs/openai-models.md b/docs/openai-models.md index 7b4a8fa1a..31f54e3c1 100644 --- a/docs/openai-models.md +++ b/docs/openai-models.md @@ -72,9 +72,7 @@ OpenAI Completion: gpt-3.5-turbo-instruct (aliases: 3.5-instruct, chatgpt-instru See [the OpenAI models documentation](https://platform.openai.com/docs/models) for details of each of these. -`gpt-4o-mini` (aliased to `4o-mini`) is the least expensive model, and is the default for if you don't specify a model at all. Consult [OpenAI's model documentation](https://platform.openai.com/docs/models) for details of the other models. - -[o1-pro](https://platform.openai.com/docs/models/o1-pro) is not available through the Chat Completions API used by LLM's default OpenAI plugin. You can install the new [llm-openai-plugin](https://github.com/simonw/llm-openai-plugin) plugin to access that model. +`gpt-5.6-luna` is one of the less expensive models, and is the default for if you don't specify a model at all. Consult [OpenAI's model documentation](https://platform.openai.com/docs/models) for details of the other models. ## Model features diff --git a/docs/python-api.md b/docs/python-api.md index 06f3f4b53..26a237598 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -7,12 +7,12 @@ Understanding this API is also important for writing {ref}`plugins`. ## Basic prompt execution -To run a prompt against the `gpt-4o-mini` model, run this: +To run a prompt against the `gpt-5.6-luna` model, run this: ```python import llm -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.6-luna") # key= is optional, you can configure the key in other ways response = model.prompt( "Five surprising names for a pet pelican", @@ -26,7 +26,7 @@ If you inspect the response before it has been evaluated it will look like this: -The `llm.get_model()` function accepts model IDs or aliases. You can also omit it to use the currently configured default model, which is `gpt-4o-mini` if you have not changed the default. +The `llm.get_model()` function accepts model IDs or aliases. You can also omit it to use the currently configured default model, which is `gpt-5.6-luna` if you have not changed the default. In this example the key is set by Python code. You can also provide the key using the `OPENAI_API_KEY` environment variable, or use the `llm keys set openai` command to store it in a `keys.json` file, see {ref}`api-keys`. @@ -68,7 +68,7 @@ This example shows two attachments - one from a file path and one from a URL: ```python import llm -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.6-luna") response = model.prompt( "Describe these images", attachments=[ @@ -87,7 +87,7 @@ Use `llm.Attachment(content=b"binary image content here")` to pass binary conten You can check which attachment types (if any) a model supports using the `model.attachment_types` set: ```python -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.6-luna") print(model.attachment_types) # {'image/gif', 'image/png', 'image/jpeg', 'image/webp'} @@ -417,7 +417,7 @@ class Dog(BaseModel): name: str age: int -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.6-luna") response = model.prompt("Describe a nice dog", schema=Dog) dog = json.loads(response.text()) print(dog) @@ -498,7 +498,7 @@ print(model.prompt("Names for otters", options={"temperature": 0.2})) Models that accept API keys should take an additional `key=` parameter to their `model.prompt()` method: ```python -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.6-luna") print(model.prompt("Names for beavers", key="sk-...")) ``` @@ -539,32 +539,30 @@ You can access this JSON data as a Python dictionary using the `response.json()` import llm from pprint import pprint -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.6-luna") response = model.prompt("3 names for an otter") json_data = response.json() pprint(json_data) ``` -Here's that example output from GPT-4o mini: -```python -{'content': 'Sure! Here are three fun names for an otter:\n' - '\n' - '1. **Splash**\n' - '2. **Bubbles**\n' - '3. **Otto** \n' - '\n' - 'Feel free to mix and match or use these as inspiration!', - 'created': 1739291215, - 'finish_reason': 'stop', - 'id': 'chatcmpl-AznO31yxgBjZ4zrzBOwJvHEWgdTaf', - 'model': 'gpt-4o-mini-2024-07-18', - 'object': 'chat.completion.chunk', - 'usage': {'completion_tokens': 43, - 'completion_tokens_details': {'accepted_prediction_tokens': 0, - 'audio_tokens': 0, - 'reasoning_tokens': 0, - 'rejected_prediction_tokens': 0}, - 'prompt_tokens': 13, - 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}, +Here's an abbreviated example of that output from GPT-5.6 Luna: +```python +{'created_at': 1785451200.0, + 'id': 'resp_...', + 'model': 'gpt-5.6-luna', + 'object': 'response', + 'output': [ + # Reasoning and message items are abbreviated here + {'content': [{'annotations': [], + 'text': '1. Splash\n2. Bubbles\n3. Otto', + 'type': 'output_text'}], + 'id': 'msg_...', + 'role': 'assistant', + 'status': 'completed', + 'type': 'message'}], + 'status': 'completed', + 'usage': {'input_tokens': 13, + 'output_tokens': 43, + 'output_tokens_details': {'reasoning_tokens': 0}, 'total_tokens': 56}} ``` @@ -918,7 +916,7 @@ Example usage: ```python import llm -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.6-luna") response = model.prompt("a poem about a hippo") response.on_done(lambda response: print(response.usage())) print(response.text()) @@ -935,7 +933,7 @@ Or using an `asyncio` model, where you need to `await response.on_done(done)` to import asyncio, llm async def run(): - model = llm.get_async_model("gpt-4o-mini") + model = llm.get_async_model("gpt-5.6-luna") response = model.prompt("a short poem about a brick") async def done(response): print(await response.usage()) @@ -957,7 +955,7 @@ The `llm.set_alias()` function can be used to define a new alias: ```python import llm -llm.set_alias("mini", "gpt-4o-mini") +llm.set_alias("mini", "gpt-5.6-luna") ``` The second argument can be a model identifier or another alias, in which case that alias will be resolved. @@ -982,12 +980,12 @@ This sets the default model to the given model ID or alias. Any changes to defau ```python import llm -llm.set_default_model("claude-3.5-sonnet") +llm.set_default_model("claude-5-sonnet") ``` ### get_default_model() -This returns the currently configured default model, or `gpt-4o-mini` if no default has been set. +This returns the currently configured default model, or `gpt-5.6-luna` if no default has been set. ```python import llm diff --git a/docs/schemas.md b/docs/schemas.md index 219e3d7a7..175bdb847 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -32,7 +32,7 @@ I got back Ziggy: ``` The response matched my schema, with `name` and `one_sentence_bio` string columns and an integer for `age`. -We're using the default LLM model here - `gpt-4o-mini`. Add `-m model` to use another model - for example use `-m o3-mini` to have O3 mini invent some dogs. +We're using the default LLM model here - `gpt-5.6-luna`. Add `-m model` to use another model - for example use `-m gpt-5.6-sol` to have GPT-5.6 Sol invent some dogs. For a list of available models that support schemas, run this command: ```bash @@ -596,4 +596,4 @@ Output: ``` If a row already has a property called `"conversation_id"` or `"response_id"` additional underscores will be appended to the ID key until it no longer overlaps with the existing keys. -The `--id-gt $ID` and `--id-gte $ID` options can be useful for ignoring logged schema data prior to a certain point, see {ref}`logging-filter-id` for details. \ No newline at end of file +The `--id-gt $ID` and `--id-gte $ID` options can be useful for ignoring logged schema data prior to a certain point, see {ref}`logging-filter-id` for details. diff --git a/docs/setup.md b/docs/setup.md index 72801ba68..2460a1014 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -167,7 +167,7 @@ You can configure LLM in a number of different ways. (setup-default-model)= ### Setting a custom default model -The model used when calling `llm` without the `-m/--model` option defaults to `gpt-4o-mini` - the fastest and least expensive OpenAI model. +The model used when calling `llm` without the `-m/--model` option defaults to `gpt-5.6-luna` - a fast and less expensive OpenAI model. You can use the `llm models default` command to set a different default model. For GPT-4o (slower and more expensive, but more capable) run this: @@ -205,4 +205,4 @@ Or turn it back on again with: ``` llm logs on ``` -Run `llm logs status` to see the current states of the setting. \ No newline at end of file +Run `llm logs status` to see the current states of the setting. diff --git a/docs/templates.md b/docs/templates.md index 3757af4b7..0a2cc93a1 100644 --- a/docs/templates.md +++ b/docs/templates.md @@ -335,7 +335,7 @@ extract: true ### Setting a default model for a template -Templates executed using `llm -t template-name` will execute using the default model that the user has configured for the tool - or `gpt-3.5-turbo` if they have not configured their own default. +Templates executed using `llm -t template-name` will execute using the default model that the user has configured for the tool - or `gpt-5.6-luna` if they have not configured their own default. You can specify a new default model for a template using the `model:` key in the associated YAML. Here's a template called `roast`: @@ -384,4 +384,4 @@ The `-sL` flags to `curl` are used to follow redirects and suppress progress met This command will fetch the content of the LLM index page and feed it to the template defined by [summarize.yaml](https://github.com/simonw/llm-templates/blob/main/summarize.yaml) in the [simonw/llm-templates](https://github.com/simonw/llm-templates) GitHub repository. -If two template loader plugins attempt to register the same prefix one of them will have `_1` added to the end of their prefix. Use `llm templates loaders` to check if this has occurred. \ No newline at end of file +If two template loader plugins attempt to register the same prefix one of them will have `_1` added to the end of their prefix. Use `llm templates loaders` to check if this has occurred. diff --git a/docs/usage.md b/docs/usage.md index 6f6e82e4f..00461354f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -6,7 +6,7 @@ The command to run a prompt is `llm prompt 'your prompt'`. This is the default c (usage-executing-prompts)= ## Executing a prompt -These examples use the default OpenAI `gpt-4o-mini` model, which requires you to first {ref}`set an OpenAI API key `. +These examples use the default OpenAI `gpt-5.6-luna` model, which requires you to first {ref}`set an OpenAI API key `. You can {ref}`install LLM plugins ` to use models from other providers, including openly licensed models you can run directly on your own computer. @@ -18,22 +18,21 @@ To disable streaming and only return the response once it has completed: ```bash llm 'Ten names for cheesecakes' --no-stream ``` -To switch from ChatGPT 4o-mini (the default) to GPT-4o: +To switch from the default model to GPT-5.6 Sol: ```bash -llm 'Ten names for cheesecakes' -m gpt-4o +llm 'Ten names for cheesecakes' -m gpt-5.6-sol ``` -You can use `-m 4o` as an even shorter shortcut. Pass `--model ` to use a different model. Run `llm models` to see a list of available models. Or if you know the name is too long to type, use `-q` once or more to provide search terms - the model with the shortest model ID that matches all of those terms (as a lowercase substring) will be used: ```bash -llm 'Ten names for cheesecakes' -q 4o -q mini +llm 'Ten names for cheesecakes' -q gpt -q sol ``` To change the default model for the current session, set the `LLM_MODEL` environment variable: ```bash -export LLM_MODEL=gpt-4.1-mini -llm 'Ten names for cheesecakes' # Uses gpt-4.1-mini +export LLM_MODEL=gpt-4.1 +llm 'Ten names for cheesecakes' # Uses gpt-4.1 ``` You can send a prompt directly to standard input like this: @@ -253,7 +252,7 @@ llm 'Five names for a pet pelican' --json [ { "id": "01jm8ec74wxsdatyn5pq1fp0s5", - "model": "gpt-4o-mini", + "model": "gpt-5.6-luna", "resolved_model": null, "prompt": "Five names for a pet pelican", "system": null, @@ -269,7 +268,7 @@ llm 'Five names for a pet pelican' --json "output_tokens": 62, "token_details": null, "conversation_name": "Five names for a pet pelican", - "conversation_model": "gpt-4o-mini", + "conversation_model": "gpt-5.6-luna", "schema_json": null, "prompt_fragments": [], "system_fragments": [], @@ -312,7 +311,7 @@ llm --schema '{ } } } -}' -m gpt-4o-mini 'invent two dogs' +}' -m gpt-5.6-sol 'invent two dogs' ``` Or use LLM's custom {ref}`concise schema syntax ` like this: @@ -453,26 +452,26 @@ llm chat -c For models that support them, you can pass options using `-o/--option`: ```bash -llm chat -m gpt-4 -o temperature 0.5 +llm chat -m gpt-4.1 -o temperature 0.5 ``` You can pass a system prompt to be used for your chat conversation: ```bash -llm chat -m gpt-4 -s 'You are a sentient cheesecake' +llm chat -m gpt-5.6-luna -s 'You are a sentient cheesecake' ``` You can also pass {ref}`a template ` - useful for creating chat personas that you wish to return to. -Here's how to create a template for your GPT-4 powered cheesecake: +Here's how to create a template for your GPT-5.6 Luna powered cheesecake: ```bash -llm --system 'You are a sentient cheesecake' -m gpt-4 --save cheesecake +llm --system 'You are a sentient cheesecake' -m gpt-5.6-luna --save cheesecake ``` Now you can start a new chat with your cheesecake any time you like using this: ```bash llm chat -t cheesecake ``` ``` -Chatting with gpt-4 +Chatting with gpt-5.6-luna Type 'exit' or 'quit' to exit Type '!multi' to enter multiple lines, then '!end' to finish Type '!edit' to open your default editor and modify the prompt @@ -494,7 +493,7 @@ To do that, type `!multi` to start a multi-line input. Type or paste your text, If your pasted text might itself contain a `!end` line, you can set a custom delimiter using `!multi abc` followed by `!end abc` at the end: ``` -Chatting with gpt-4 +Chatting with gpt-5.6-luna Type 'exit' or 'quit' to exit Type '!multi' to enter multiple lines, then '!end' to finish Type '!edit' to open your default editor and modify the prompt. @@ -514,7 +513,7 @@ urllib.error.URLError: `: ```bash -llm models options show gpt-4o +llm models options show gpt-4.1 ``` To clear a default option, use the `llm models options clear` command: ```bash -llm models options clear gpt-4o temperature +llm models options clear gpt-4.1 temperature ``` Or clear all default options for a model like this: ```bash -llm models options clear gpt-4o +llm models options clear gpt-4.1 ``` Default model options are respected by both the `llm prompt` and the `llm chat` commands. They will not be applied when you use LLM as a {ref}`Python library `. diff --git a/llm/__init__.py b/llm/__init__.py index 1bdfea4fc..18207550a 100644 --- a/llm/__init__.py +++ b/llm/__init__.py @@ -86,7 +86,7 @@ "user", "user_dir", ] -DEFAULT_MODEL = "gpt-4o-mini" +DEFAULT_MODEL = "gpt-5.6-luna" def get_plugins(all=False): diff --git a/llm/cli.py b/llm/cli.py index 18f2746ec..fcaa1b293 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -570,7 +570,7 @@ def prompt( \b llm 'Capital of France?' - llm 'Capital of France?' -m gpt-4o + llm 'Capital of France?' -m gpt-5.5 llm 'Capital of France?' -s 'answer in Spanish' Multi-modal models can be called with attachments like this: @@ -2812,13 +2812,13 @@ def aliases_set(alias, model_id, query): Example usage: \b - llm aliases set mini gpt-4o-mini + llm aliases set luna gpt-5.6-luna Alternatively you can omit the model ID and specify one or more -q options. The first model matching all of those query strings will be used. \b - llm aliases set mini -q 4o -q mini + llm aliases set luna -q gpt -q luna """ if not model_id: if not query: @@ -3704,7 +3704,7 @@ def options_show(model): Example usage: \b - llm models options show gpt-4o + llm models options show gpt-4.1 """ import llm @@ -3736,7 +3736,7 @@ def options_set(model, key, value): Example usage: \b - llm models options set gpt-4o temperature 0.5 + llm models options set gpt-4.1 temperature 0.5 """ import llm @@ -3771,9 +3771,9 @@ def options_clear(model, key): Example usage: \b - llm models options clear gpt-4o + llm models options clear gpt-4.1 # Or for a single option - llm models options clear gpt-4o temperature + llm models options clear gpt-4.1 temperature """ import llm diff --git a/tests/conftest.py b/tests/conftest.py index 1b7af85dc..a4b6add12 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -262,6 +262,56 @@ def mocked_openai_chat(httpx_mock): return httpx_mock +@pytest.fixture +def mock_openai_responses(httpx_mock): + def add_response( + text="Bob, Alice, Eve", + model="gpt-5.6-luna", + response_id="resp_test", + message_id="msg_test", + ): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": response_id, + "object": "response", + "created_at": 1, + "model": model, + "output": [ + { + "type": "message", + "id": message_id, + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": text, + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + }, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + + return add_response + + +@pytest.fixture +def mocked_openai_responses(httpx_mock, mock_openai_responses): + mock_openai_responses() + return httpx_mock + + @pytest.fixture def mocked_openai_chat_returning_fenced_code(httpx_mock): httpx_mock.add_response( diff --git a/tests/test_fragments_cli.py b/tests/test_fragments_cli.py index 66262cc61..b3ad42aa7 100644 --- a/tests/test_fragments_cli.py +++ b/tests/test_fragments_cli.py @@ -154,7 +154,16 @@ def test_fragment_url_user_agent(mocked_openai_chat, user_path): text="Hello from URL", ) runner = CliRunner() - result = runner.invoke(cli, ["prompt", "-f", "https://example.com/fragment.txt"]) + result = runner.invoke( + cli, + [ + "prompt", + "-m", + "gpt-4o-mini", + "-f", + "https://example.com/fragment.txt", + ], + ) assert result.exit_code == 0 # Verify the User-Agent header was sent for the fragment URL request diff --git a/tests/test_keys.py b/tests/test_keys.py index 1cb71b430..7668479c7 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -88,27 +88,39 @@ def assert_key(key): runner = CliRunner() # Called without --key uses stored key - result = runner.invoke(cli, ["hello", "--no-stream"], catch_exceptions=False) + result = runner.invoke( + cli, + ["hello", "--no-stream", "-m", "gpt-4o-mini"], + catch_exceptions=False, + ) assert result.exit_code == 0 assert_key("from-keys-file") # Called without --key and without keys.json uses environment variable keys_path.write_text("{}", "utf-8") - result2 = runner.invoke(cli, ["hello", "--no-stream"], catch_exceptions=False) + result2 = runner.invoke( + cli, + ["hello", "--no-stream", "-m", "gpt-4o-mini"], + catch_exceptions=False, + ) assert result2.exit_code == 0 assert_key("from-env") keys_path.write_text(json.dumps(KEYS), "utf-8") # Called with --key name-in-keys.json uses that value result3 = runner.invoke( - cli, ["hello", "--key", "other", "--no-stream"], catch_exceptions=False + cli, + ["hello", "--key", "other", "--no-stream", "-m", "gpt-4o-mini"], + catch_exceptions=False, ) assert result3.exit_code == 0 assert_key("other-key") # Called with --key something-else uses exactly that result4 = runner.invoke( - cli, ["hello", "--key", "custom-key", "--no-stream"], catch_exceptions=False + cli, + ["hello", "--key", "custom-key", "--no-stream", "-m", "gpt-4o-mini"], + catch_exceptions=False, ) assert result4.exit_code == 0 assert_key("custom-key") diff --git a/tests/test_llm.py b/tests/test_llm.py index 633fc1a4c..0a201e7ef 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -24,7 +24,7 @@ def test_version(): @pytest.mark.parametrize("custom_database_path", (False, True)) def test_llm_prompt_creates_log_database( - mocked_openai_chat, tmpdir, monkeypatch, custom_database_path + mocked_openai_responses, tmpdir, monkeypatch, custom_database_path ): user_path = tmpdir / "user" custom_db_path = tmpdir / "custom_log.db" @@ -60,7 +60,7 @@ def test_llm_prompt_creates_log_database( ), ) def test_llm_default_prompt( - mocked_openai_chat, use_stdin, user_path, logs_off, logs_args, should_log + mocked_openai_responses, use_stdin, user_path, logs_off, logs_args, should_log ): # Reset the log_path database log_path = user_path / "logs.db" @@ -96,7 +96,7 @@ def test_llm_default_prompt( result = runner.invoke(cli, args, input=input, catch_exceptions=False) assert result.exit_code == 0 assert result.output == "Bob, Alice, Eve\n" - last_request = mocked_openai_chat.get_requests()[-1] + last_request = mocked_openai_responses.get_requests()[-1] assert last_request.headers["Authorization"] == "Bearer X" # Was it logged? The legacy tables are read-only now, so the turn @@ -110,7 +110,7 @@ def test_llm_default_prompt( assert len(rows) == 1 row = rows[0] - assert row["model"] == "gpt-4o-mini" + assert row["model"] == "gpt-5.6-luna" assert isinstance(row["duration_ms"], int) assert isinstance(row["datetime_utc"], str) @@ -124,7 +124,7 @@ def test_llm_default_prompt( assert ( log_json[0].items() >= { - "model": "gpt-4o-mini", + "model": "gpt-5.6-luna", "prompt": "three names \nfor a pet pelican", "system": None, # prompt_json and response_json are no longer recorded: the @@ -134,33 +134,23 @@ def test_llm_default_prompt( "response": "Bob, Alice, Eve", # This doesn't have the \n after three names: "conversation_name": "three names for a pet pelican", - "conversation_model": "gpt-4o-mini", + "conversation_model": "gpt-5.6-luna", }.items() ) @mock.patch.dict(os.environ, {"OPENAI_API_KEY": "X"}) @pytest.mark.parametrize("async_", (False, True)) -def test_llm_prompt_continue(httpx_mock, user_path, async_): - httpx_mock.add_response( - method="POST", - url="https://api.openai.com/v1/chat/completions", - json={ - "model": "gpt-4o-mini", - "usage": {}, - "choices": [{"message": {"content": "Bob, Alice, Eve"}}], - }, - headers={"Content-Type": "application/json"}, +def test_llm_prompt_continue(httpx_mock, mock_openai_responses, user_path, async_): + mock_openai_responses( + text="Bob, Alice, Eve", + response_id="resp_first", + message_id="msg_first", ) - httpx_mock.add_response( - method="POST", - url="https://api.openai.com/v1/chat/completions", - json={ - "model": "gpt-4o-mini", - "usage": {}, - "choices": [{"message": {"content": "Terry"}}], - }, - headers={"Content-Type": "application/json"}, + mock_openai_responses( + text="Terry", + response_id="resp_second", + message_id="msg_second", ) log_path = user_path / "logs.db" @@ -571,8 +561,8 @@ def test_model_defaults(tmpdir, monkeypatch): monkeypatch.setenv("LLM_USER_PATH", user_dir) config_path = pathlib.Path(user_dir) / "default_model.txt" assert not config_path.exists() - assert llm.get_default_model() == "gpt-4o-mini" - assert llm.get_model().model_id == "gpt-4o-mini" + assert llm.get_default_model() == "gpt-5.6-luna" + assert llm.get_model().model_id == "gpt-5.6-luna" llm.set_default_model("gpt-4o") assert config_path.exists() assert llm.get_default_model() == "gpt-4o" @@ -821,27 +811,22 @@ def test_schemas_dsl(): @mock.patch.dict(os.environ, {"OPENAI_API_KEY": "X"}) @pytest.mark.parametrize("custom_database_path", (False, True)) def test_llm_prompt_continue_with_database( - tmpdir, monkeypatch, httpx_mock, user_path, custom_database_path + tmpdir, + monkeypatch, + httpx_mock, + mock_openai_responses, + user_path, + custom_database_path, ): - httpx_mock.add_response( - method="POST", - url="https://api.openai.com/v1/chat/completions", - json={ - "model": "gpt-4o-mini", - "usage": {}, - "choices": [{"message": {"content": "Bob, Alice, Eve"}}], - }, - headers={"Content-Type": "application/json"}, + mock_openai_responses( + text="Bob, Alice, Eve", + response_id="resp_first", + message_id="msg_first", ) - httpx_mock.add_response( - method="POST", - url="https://api.openai.com/v1/chat/completions", - json={ - "model": "gpt-4o-mini", - "usage": {}, - "choices": [{"message": {"content": "Terry"}}], - }, - headers={"Content-Type": "application/json"}, + mock_openai_responses( + text="Terry", + response_id="resp_second", + message_id="msg_second", ) user_path = tmpdir / "user" diff --git a/tests/test_templates.py b/tests/test_templates.py index 9ef2d8888..9a01b7c5e 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -201,7 +201,7 @@ def test_templates_error_on_missing_schema(templates_path): ( "'Summarize this: $input'", "Input text", - [], + ["-m", "gpt-4o-mini"], "gpt-4o-mini", "Summarize this: Input text", None, @@ -229,7 +229,7 @@ def test_templates_error_on_missing_schema(templates_path): pytest.param( "boo", "Input text", - ["-s", "custom system"], + ["-m", "gpt-4o-mini", "-s", "custom system"], "gpt-4o-mini", [ {"role": "system", "content": "custom system"}, @@ -253,7 +253,7 @@ def test_templates_error_on_missing_schema(templates_path): ( "prompt: 'Say $hello'", "Input text", - ["-p", "hello", "Blah"], + ["-m", "gpt-4o-mini", "-p", "hello", "Blah"], "gpt-4o-mini", "Say Blah\nInput text", None, @@ -262,7 +262,7 @@ def test_templates_error_on_missing_schema(templates_path): ( "prompt: 'Say pelican'", "", - [], + ["-m", "gpt-4o-mini"], "gpt-4o-mini", "Say pelican", None, @@ -272,7 +272,7 @@ def test_templates_error_on_missing_schema(templates_path): ( "system: 'Summarize this'", "Input text", - [], + ["-m", "gpt-4o-mini"], "gpt-4o-mini", [ {"content": "Summarize this", "role": "system"}, @@ -285,7 +285,7 @@ def test_templates_error_on_missing_schema(templates_path): ( "prompt: 'Summarize this: $input'\noptions:\n temperature: 0.5", "Input text", - [], + ["-m", "gpt-4o-mini"], "gpt-4o-mini", "Summarize this: Input text", None, @@ -295,7 +295,7 @@ def test_templates_error_on_missing_schema(templates_path): ( "prompt: 'Summarize this: $input'\noptions:\n temperature: 0.5", "Input text", - ["-o", "temperature", "0.7"], + ["-m", "gpt-4o-mini", "-o", "temperature", "0.7"], "gpt-4o-mini", "Summarize this: Input text", None, From 859cea1e67c52df620d543ee84715e600f51956d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 13:32:03 -0700 Subject: [PATCH 184/258] Require --chat for endpoint interactive mode --- docs/help.md | 12 +++++------- docs/other-models.md | 9 ++++++--- llm/default_plugins/openai_models.py | 24 ++++++++++++++---------- tests/test_openai_endpoint.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 20 deletions(-) diff --git a/docs/help.md b/docs/help.md index 085c18847..385bc91a5 100644 --- a/docs/help.md +++ b/docs/help.md @@ -1083,11 +1083,10 @@ Usage: llm openai endpoint [OPTIONS] URL [PROMPT] Run against an OpenAI-compatible endpoint without logging. - If PROMPT is provided, execute it once. If PROMPT is omitted in an interactive - terminal, start a chat unless --template is provided. Templates run once by - default; use --chat to apply one interactively. Piped stdin is treated as a - one-off prompt. Use --models to list the available model IDs without running a - prompt. + PROMPT or stdin is executed once. If neither is provided, wait for input on + stdin. Use --chat to start an interactive chat. Templates run once by default; + use --chat to apply one interactively. Use --models to list the available + model IDs without running a prompt. Options: -m, --model TEXT Model ID (required unless --models or provided @@ -1111,8 +1110,7 @@ Options: -H, --header ... Additional HTTP header --responses Use the Responses API instead of Chat Completions - --chat Start an interactive chat, even when stdin is - not a terminal + --chat Start an interactive chat --models List model IDs from the endpoint and exit --no-stream Do not stream output -R, --hide-reasoning Hide reasoning output diff --git a/docs/other-models.md b/docs/other-models.md index 5a7f257ef..cbca68b2f 100644 --- a/docs/other-models.md +++ b/docs/other-models.md @@ -59,14 +59,17 @@ will request `/v1/models`: llm openai endpoint https://example.com/v1 --models ``` -Omit the prompt to start an interactive chat: +Omit the prompt to read it from stdin. In an interactive terminal the command waits for input until EOF, matching `llm prompt`: ```bash llm openai endpoint https://example.com/v1 -m model-id ``` -Piped stdin is treated as a one-off prompt. Use `--chat` to explicitly start -an interactive chat when stdin is not a terminal. +Use `--chat` to start an interactive chat: + +```bash +llm openai endpoint https://example.com/v1 -m model-id --chat +``` Use `-a` or `--attachment` to attach an image or PDF. Chat Completions endpoints can also receive WAV or MP3 audio attachments: diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 72863efe3..268693954 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -472,7 +472,7 @@ def openai_(): "force_chat", "--chat", is_flag=True, - help="Start an interactive chat, even when stdin is not a terminal", + help="Start an interactive chat", ) @click.option( "list_models", @@ -508,11 +508,10 @@ def endpoint( """ Run against an OpenAI-compatible endpoint without logging. - If PROMPT is provided, execute it once. If PROMPT is omitted in an - interactive terminal, start a chat unless --template is provided. - Templates run once by default; use --chat to apply one interactively. - Piped stdin is treated as a one-off prompt. Use --models to list the - available model IDs without running a prompt. + PROMPT or stdin is executed once. If neither is provided, wait for + input on stdin. Use --chat to start an interactive chat. Templates run + once by default; use --chat to apply one interactively. Use --models + to list the available model IDs without running a prompt. """ from llm.cli import ( AttachmentError, @@ -598,16 +597,13 @@ def endpoint( tools, python_tools, tools_debug, tools_approve, chain_limit ) resolved_attachments = [*attachments, *attachment_types] - is_chat = force_chat or ( - prompt is None and template_obj is None and sys.stdin.isatty() - ) try: if list_models: for available_model in model.get_client(key).models.list(): click.echo(available_model.id) return - if is_chat: + if force_chat: conversation = model.conversation() def transform_chat_prompt(chat_prompt): @@ -648,6 +644,14 @@ def execute_chat_prompt(chat_prompt, _fragments, turn_attachments): prompt = " ".join( part for part in (stdin_prompt, prompt) if part is not None ) + elif ( + prompt is None + and not resolved_attachments + and (template_obj is None or "input" in template_obj.vars()) + ): + # Match `llm prompt`: wait for stdin until EOF instead of + # implicitly starting an interactive chat. + prompt = sys.stdin.read() if template_obj: prompt, system = _apply_template(template_obj, prompt, params, system) if prompt is None: diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py index 037403dea..177649ae1 100644 --- a/tests/test_openai_endpoint.py +++ b/tests/test_openai_endpoint.py @@ -657,6 +657,34 @@ def test_endpoint_reads_one_off_prompt_from_stdin(httpx_mock, user_path): assert not (user_path / "logs.db").exists() +def test_endpoint_without_prompt_waits_for_stdin(httpx_mock, user_path, monkeypatch): + base_url = "https://terminal-stdin.example.test/v1" + _add_chat_response(httpx_mock, base_url, "From awaited stdin") + monkeypatch.setattr("click.testing._NamedTextIOWrapper.isatty", lambda self: True) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "-m", + "test-model", + "--no-stream", + ], + input="Hello after waiting for stdin", + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "From awaited stdin\n" + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["messages"] == [ + {"role": "user", "content": "Hello after waiting for stdin"} + ] + assert not (user_path / "logs.db").exists() + + def test_endpoint_interactive_chat_preserves_history( httpx_mock, user_path, templates_path ): From 03a33e36e17dfa330c2bb9f308e6efdda9ffa217 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 13:37:33 -0700 Subject: [PATCH 185/258] Expose optional endpoint model capabilities --- llm/default_plugins/openai_models.py | 4 ++++ tests/test_openai_endpoint.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 268693954..a71d4ce97 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -567,6 +567,10 @@ def endpoint( headers=dict(headers), vision=True, audio=not use_responses, + # Optimistically expose capabilities that have no effect until + # the user explicitly exercises them. + verbosity=True, + image_detail_original=True, supports_tools=True, ) diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py index 177649ae1..d073f6371 100644 --- a/tests/test_openai_endpoint.py +++ b/tests/test_openai_endpoint.py @@ -516,6 +516,9 @@ def test_endpoint_responses_api(httpx_mock, user_path): "test-model", "--responses", "--no-stream", + "-o", + "verbosity", + "low", ], catch_exceptions=False, ) @@ -528,6 +531,7 @@ def test_endpoint_responses_api(httpx_mock, user_path): "model": "test-model", "store": False, "stream": False, + "text": {"verbosity": "low"}, } @@ -554,6 +558,9 @@ def test_endpoint_responses_api_attachment(httpx_mock, user_path): "--at", "https://images.example.test/test.jpg", "image/jpeg", + "-o", + "image_detail", + "original", ], catch_exceptions=False, ) @@ -569,6 +576,7 @@ def test_endpoint_responses_api_attachment(httpx_mock, user_path): { "type": "input_image", "image_url": "https://images.example.test/test.jpg", + "detail": "original", }, ], } From ab4b8405a784b5597666e60cc5a14e97a46a1072 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 13:48:40 -0700 Subject: [PATCH 186/258] Expose reasoning effort for endpoint models Refs https://github.com/simonw/llm/pull/1568/changes#r3686136451 --- docs/other-models.md | 11 ++++++++ llm/default_plugins/openai_models.py | 40 ++++++++++++++++++---------- tests/test_openai_endpoint.py | 14 +++++++++- 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/docs/other-models.md b/docs/other-models.md index cbca68b2f..7b2806b34 100644 --- a/docs/other-models.md +++ b/docs/other-models.md @@ -101,6 +101,17 @@ interactive chat started using `--chat`, the template is applied to each turn. Without `--chat`, a template that provides its own prompt runs once even when no prompt argument is supplied. +Reasoning-capable endpoints can be given a reasoning effort using `-o reasoning_effort`, with a value of `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, or `max`: + +```bash +llm openai endpoint https://example.com/v1 \ + -m model-id \ + -o reasoning_effort high \ + "Solve this problem" +``` + +The command does not send reasoning-specific request fields by default and does not request a reasoning summary. Those fields are only added when `reasoning_effort` is used. An endpoint that does not support the option will return its own API error. + Use `-T` or `--tool` to make an installed LLM tool available to the model, or `--functions` to load Python functions from an inline code block or file: diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index a71d4ce97..d190b7e86 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -560,19 +560,23 @@ def endpoint( ) model_class = Responses if use_responses else Chat - model = model_class( - model_id=model_id or "", - model_name=model_id or "", - api_base=url, - headers=dict(headers), - vision=True, - audio=not use_responses, + model_kwargs = { + "model_id": model_id or "", + "model_name": model_id or "", + "api_base": url, + "headers": dict(headers), + "vision": True, + "audio": not use_responses, # Optimistically expose capabilities that have no effect until # the user explicitly exercises them. - verbosity=True, - image_detail_original=True, - supports_tools=True, - ) + "reasoning": True, + "verbosity": True, + "image_detail_original": True, + "supports_tools": True, + } + if use_responses: + model_kwargs["reasoning_summary"] = False + model = model_class(**model_kwargs) # A configured api_base never receives the user's default OpenAI key. # Match that safety property here: only send credentials when --key @@ -1600,7 +1604,7 @@ def _build_responses_kwargs(self, prompt, stream): kwargs["seed"] = seed if self._reasoning: reasoning = {} - if not getattr(prompt, "hide_reasoning", False): + if self._reasoning_summary and not getattr(prompt, "hide_reasoning", False): reasoning["summary"] = "auto" if reasoning_effort: reasoning["effort"] = reasoning_effort @@ -1726,6 +1730,7 @@ def __init__( supports_schema=False, supports_tools=False, allows_system_prompt=True, + reasoning_summary=True, ): super().__init__( model_id, @@ -1747,6 +1752,7 @@ def __init__( allows_system_prompt=allows_system_prompt, ) self._reasoning = reasoning + self._reasoning_summary = reasoning_summary self._verbosity = verbosity self._image_detail_original = image_detail_original # Override the Options class so that ``-o chat_completions 1`` is @@ -1784,7 +1790,9 @@ def execute( if instructions is not None: kwargs["instructions"] = instructions kwargs["store"] = False - if self._reasoning: + if self._reasoning and ( + self._reasoning_summary or getattr(prompt.options, "reasoning_effort", None) + ): kwargs["include"] = ["reasoning.encrypted_content"] client = self.get_client(key) @@ -1955,6 +1963,7 @@ def __init__( supports_schema=False, supports_tools=False, allows_system_prompt=True, + reasoning_summary=True, ): super().__init__( model_id, @@ -1976,6 +1985,7 @@ def __init__( allows_system_prompt=allows_system_prompt, ) self._reasoning = reasoning + self._reasoning_summary = reasoning_summary self._verbosity = verbosity self._image_detail_original = image_detail_original self.Options = build_options_class( @@ -2014,7 +2024,9 @@ async def execute( if instructions is not None: kwargs["instructions"] = instructions kwargs["store"] = False - if self._reasoning: + if self._reasoning and ( + self._reasoning_summary or getattr(prompt.options, "reasoning_effort", None) + ): kwargs["include"] = ["reasoning.encrypted_content"] client = self.get_client(key, async_=True) diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py index d073f6371..bd8c23387 100644 --- a/tests/test_openai_endpoint.py +++ b/tests/test_openai_endpoint.py @@ -174,6 +174,9 @@ def test_endpoint_chat_completions_does_not_log_or_leak_default_key( "-H", "X-Test", "one", + "-o", + "reasoning_effort", + "low", ], catch_exceptions=False, ) @@ -188,6 +191,7 @@ def test_endpoint_chat_completions_does_not_log_or_leak_default_key( assert json.loads(request.content) == { "messages": [{"role": "user", "content": "Hello"}], "model": "test-model", + "reasoning_effort": "low", "stream": False, } @@ -519,6 +523,9 @@ def test_endpoint_responses_api(httpx_mock, user_path): "-o", "verbosity", "low", + "-o", + "reasoning_effort", + "low", ], catch_exceptions=False, ) @@ -528,7 +535,9 @@ def test_endpoint_responses_api(httpx_mock, user_path): assert not (user_path / "logs.db").exists() assert json.loads(httpx_mock.get_requests()[0].content) == { "input": [{"role": "user", "content": "Hello"}], + "include": ["reasoning.encrypted_content"], "model": "test-model", + "reasoning": {"effort": "low"}, "store": False, "stream": False, "text": {"verbosity": "low"}, @@ -568,7 +577,10 @@ def test_endpoint_responses_api_attachment(httpx_mock, user_path): assert result.exit_code == 0 assert result.output == "A remote image\n" assert not (user_path / "logs.db").exists() - assert json.loads(httpx_mock.get_requests()[0].content)["input"] == [ + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert "include" not in request_body + assert "reasoning" not in request_body + assert request_body["input"] == [ { "role": "user", "content": [ From 2700e4d6cb7a09d84c814751423d7969be856055 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 14:00:55 -0700 Subject: [PATCH 187/258] Show error if call to llm openai endpoint --models fails https://github.com/simonw/llm/pull/1568#issuecomment-5136144202 --- llm/default_plugins/openai_models.py | 8 +++++++- tests/test_openai_endpoint.py | 20 ++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index d190b7e86..154e5dd2a 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -607,7 +607,13 @@ def endpoint( resolved_attachments = [*attachments, *attachment_types] try: if list_models: - for available_model in model.get_client(key).models.list(): + available_models = model.get_client(key).models.list() + error = getattr(available_models, "error", None) + if error: + if isinstance(error, dict): + error = error.get("message") or json.dumps(error) + raise click.ClickException(str(error)) + for available_model in available_models: click.echo(available_model.id) return diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py index bd8c23387..3c3c5c444 100644 --- a/tests/test_openai_endpoint.py +++ b/tests/test_openai_endpoint.py @@ -500,6 +500,26 @@ def test_endpoint_lists_models_without_model_or_logging( assert request.headers["X-Test"] == "one" +def test_endpoint_models_surfaces_error_from_successful_response(httpx_mock, user_path): + base_url = "https://models-error.example.test" + httpx_mock.add_response( + method="GET", + url=f"{base_url}/models", + json={"error": "Unexpected endpoint or method. (GET /models)"}, + headers={"Content-Type": "application/json"}, + ) + + result = CliRunner().invoke( + cli, + ["openai", "endpoint", base_url, "--models"], + catch_exceptions=False, + ) + + assert result.exit_code == 1 + assert result.output == "Error: Unexpected endpoint or method. (GET /models)\n" + assert not (user_path / "logs.db").exists() + + def test_endpoint_responses_api(httpx_mock, user_path): base_url = "https://responses.example.test/v1" httpx_mock.add_response( From 3c85ff690e861a8582307749c5272f08ad1e0e2a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 14:03:01 -0700 Subject: [PATCH 188/258] Unwrap the docs --- docs/other-models.md | 40 ++++++++++------------------------------ 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/docs/other-models.md b/docs/other-models.md index 7b2806b34..fc30223cc 100644 --- a/docs/other-models.md +++ b/docs/other-models.md @@ -30,8 +30,7 @@ Projects such as [LocalAI](https://localai.io/) offer a REST API that imitates t ### Run against an endpoint without configuring it -Use `llm openai endpoint` to run a prompt directly against an -OpenAI-compatible base URL: +Use `llm openai endpoint` to run a prompt directly against an OpenAI-compatible base URL: ```bash llm openai endpoint https://example.com/v1 \ @@ -39,10 +38,7 @@ llm openai endpoint https://example.com/v1 \ "What is the capital of France?" ``` -This command does not register the model and does not log the prompt or -response. It also does not send your configured OpenAI API key to the endpoint. -Use `--key` to explicitly provide a key or the alias of a key saved using -`llm keys set`: +This command does not register the model and does not log the prompt or response. It also does not send your configured OpenAI API key to the endpoint. Use `--key` to explicitly provide a key or the alias of a key saved using `llm keys set`: ```bash llm openai endpoint https://example.com/v1 \ @@ -51,9 +47,7 @@ llm openai endpoint https://example.com/v1 \ "What is the capital of France?" ``` -List the model IDs advertised by the endpoint using `--models`. This requests -the `models` resource relative to the base URL, so a base URL ending in `/v1` -will request `/v1/models`: +List the model IDs advertised by the endpoint using `--models`. This requests the `models` resource relative to the base URL, so a base URL ending in `/v1` will request `/v1/models`: ```bash llm openai endpoint https://example.com/v1 --models @@ -71,8 +65,7 @@ Use `--chat` to start an interactive chat: llm openai endpoint https://example.com/v1 -m model-id --chat ``` -Use `-a` or `--attachment` to attach an image or PDF. Chat Completions -endpoints can also receive WAV or MP3 audio attachments: +Use `-a` or `--attachment` to attach an image or PDF. Chat Completions endpoints can also receive WAV or MP3 audio attachments: ```bash llm openai endpoint https://example.com/v1 \ @@ -81,13 +74,9 @@ llm openai endpoint https://example.com/v1 \ "Describe this image" ``` -Use `--at path-or-url mimetype` when the attachment type cannot be inferred. -Attachments provided when starting an interactive chat are included with the -first message. +Use `--at path-or-url mimetype` when the attachment type cannot be inferred. Attachments provided when starting an interactive chat are included with the first message. -Use `-t` or `--template` to apply an existing LLM template. Template prompts, -system prompts, defaults, model options, model IDs, and attachments are -supported. Pass template variables using `-p` or `--param`: +Use `-t` or `--template` to apply an existing LLM template. Template prompts, system prompts, defaults, model options, model IDs, and attachments are supported. Pass template variables using `-p` or `--param`: ```bash llm openai endpoint https://example.com/v1 \ @@ -96,10 +85,7 @@ llm openai endpoint https://example.com/v1 \ "Text to summarize" ``` -The `-m` option can be omitted if the template specifies a model. In an -interactive chat started using `--chat`, the template is applied to each turn. -Without `--chat`, a template that provides its own prompt runs once even when -no prompt argument is supplied. +The `-m` option can be omitted if the template specifies a model. In an interactive chat started using `--chat`, the template is applied to each turn. Without `--chat`, a template that provides its own prompt runs once even when no prompt argument is supplied. Reasoning-capable endpoints can be given a reasoning effort using `-o reasoning_effort`, with a value of `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, or `max`: @@ -112,8 +98,7 @@ llm openai endpoint https://example.com/v1 \ The command does not send reasoning-specific request fields by default and does not request a reasoning summary. Those fields are only added when `reasoning_effort` is used. An endpoint that does not support the option will return its own API error. -Use `-T` or `--tool` to make an installed LLM tool available to the model, or -`--functions` to load Python functions from an inline code block or file: +Use `-T` or `--tool` to make an installed LLM tool available to the model, or `--functions` to load Python functions from an inline code block or file: ```bash llm openai endpoint https://example.com/v1 \ @@ -123,14 +108,9 @@ llm openai endpoint https://example.com/v1 \ "What time is it?" ``` -Tool calls are executed locally and their results are sent back to the endpoint -until it returns a final answer. `--chain-limit` controls the maximum number of -responses, `--tools-debug` shows tool execution details, and `--tools-approve` -asks for confirmation before each call. Tools and trusted Python functions -declared by local templates are supported too. +Tool calls are executed locally and their results are sent back to the endpoint until it returns a final answer. `--chain-limit` controls the maximum number of responses, `--tools-debug` shows tool execution details, and `--tools-approve` asks for confirmation before each call. Tools and trusted Python functions declared by local templates are supported too. -The command uses the Chat Completions API by default. Add `--responses` for an -endpoint that implements the Responses API: +The command uses the Chat Completions API by default. Add `--responses` for an endpoint that implements the Responses API: ```bash llm openai endpoint https://example.com/v1 \ From 6977e1fccdaa19049d85c62b3c65f504ec38f6fd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 14:05:25 -0700 Subject: [PATCH 189/258] Changelog entry for openai endpoint, refs #1565 --- docs/changelog.md | 1 + docs/other-models.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 14a0df99e..10f6337d8 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,7 @@ ## Unreleased - The default model for users who have not set their own default is now [GPT-5.6 Luna](https://developers.openai.com/api/docs/models/gpt-5.6-luna). It was previously [GPT-4o mini](https://developers.openai.com/api/docs/models/gpt-4o-mini). Luna is a much better and more recent model, albeit slightly more expensive - $0.20 per million input tokens and $1.20 per million output tokens, compared to $0.15/$0.60 for 4o mini. You can switch back to 4o mini using `llm models default gpt-4o-mini`, or switch to [GPT-5 nano](https://developers.openai.com/api/docs/models/gpt-5-nano), an even cheaper default model ($0.05/$0.40), using `llm models default gpt-5-nano`. +- New {ref}`llm openai endpoint ` command for running prompts, chats and model listings against arbitrary OpenAI-compatible endpoints without first configuring a model. These calls are not logged. [#1565](https://github.com/simonw/llm/issues/1565) (v0_32_rc1)= ## 0.32rc1 diff --git a/docs/other-models.md b/docs/other-models.md index fc30223cc..ad442831c 100644 --- a/docs/other-models.md +++ b/docs/other-models.md @@ -28,6 +28,7 @@ Check the {ref}`plugin directory ` for the latest list of avai Projects such as [LocalAI](https://localai.io/) offer a REST API that imitates the OpenAI API but can be used to run other models, including models that can be installed on your own machine. These can be added using the same configuration mechanism. +(openai-endpoint)= ### Run against an endpoint without configuring it Use `llm openai endpoint` to run a prompt directly against an OpenAI-compatible base URL: From e8796845cb88f8231e477714e494464e2aa3c04e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 14:16:22 -0700 Subject: [PATCH 190/258] Schema support for llm openai endpoint, refs #1565 Demo here: https://github.com/simonw/llm/pull/1568#issuecomment-5136317329 --- docs/help.md | 2 + docs/other-models.md | 11 +- llm/default_plugins/openai_models.py | 45 ++++++- tests/test_openai_endpoint.py | 171 +++++++++++++++++++++++++++ 4 files changed, 226 insertions(+), 3 deletions(-) diff --git a/docs/help.md b/docs/help.md index 385bc91a5..c34422de0 100644 --- a/docs/help.md +++ b/docs/help.md @@ -1095,6 +1095,8 @@ Options: -t, --template TEXT Template to use -p, --param ... Parameters for template -o, --option ... key/value options for the model + --schema TEXT JSON schema, filepath or ID + --schema-multi TEXT JSON schema to use for multiple results -a, --attachment ATTACHMENT Attachment path or URL or - --at, --attachment-type ... Attachment with explicit mimetype, diff --git a/docs/other-models.md b/docs/other-models.md index ad442831c..6848c2d14 100644 --- a/docs/other-models.md +++ b/docs/other-models.md @@ -77,7 +77,7 @@ llm openai endpoint https://example.com/v1 \ Use `--at path-or-url mimetype` when the attachment type cannot be inferred. Attachments provided when starting an interactive chat are included with the first message. -Use `-t` or `--template` to apply an existing LLM template. Template prompts, system prompts, defaults, model options, model IDs, and attachments are supported. Pass template variables using `-p` or `--param`: +Use `-t` or `--template` to apply an existing LLM template. Template prompts, system prompts, defaults, model options, model IDs, schemas, and attachments are supported. Pass template variables using `-p` or `--param`: ```bash llm openai endpoint https://example.com/v1 \ @@ -88,6 +88,15 @@ llm openai endpoint https://example.com/v1 \ The `-m` option can be omitted if the template specifies a model. In an interactive chat started using `--chat`, the template is applied to each turn. Without `--chat`, a template that provides its own prompt runs once even when no prompt argument is supplied. +Use `--schema` to request structured JSON output or `--schema-multi` to request an array of matching items. These accept the same inline JSON, file paths, stored schema IDs, template references and {ref}`concise schema syntax ` as `llm prompt`: + +```bash +llm openai endpoint https://example.com/v1 \ + -m model-id \ + --schema 'name, age int' \ + "Invent a dog" +``` + Reasoning-capable endpoints can be given a reasoning effort using `-o reasoning_effort`, with a value of `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, or `max`: ```bash diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 154e5dd2a..df4ebf60b 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -9,6 +9,7 @@ import click import httpx import openai +import sqlite_utils import yaml from pydantic import Field, ValidationError, create_model, field_validator @@ -402,7 +403,12 @@ def embed_batch(self, items: Iterable[str | bytes]) -> Iterator[list[float]]: @hookimpl def register_commands(cli): - from llm.cli import AttachmentType, attachment_types_callback, tool_options + from llm.cli import ( + AttachmentType, + attachment_types_callback, + schema_option, + tool_options, + ) @cli.group(name="openai") def openai_(): @@ -435,6 +441,11 @@ def openai_(): multiple=True, help="key/value options for the model", ) + @schema_option + @click.option( + "--schema-multi", + help="JSON schema to use for multiple results", + ) @click.option( "attachments", "-a", @@ -490,6 +501,8 @@ def endpoint( template, param, options, + schema_input, + schema_multi, attachments, attachment_types, tools, @@ -524,7 +537,11 @@ def endpoint( _tool_chain_kwargs, display_stream_events, load_template, + logs_db_path, + migrate, + multi_schema, render_errors, + resolve_schema_input, ) if list_models and prompt is not None: @@ -533,9 +550,28 @@ def endpoint( raise click.ClickException("--models cannot be used with --template") if list_models and (tools or python_tools): raise click.ClickException("--models cannot be used with tools") + if list_models and (schema_input or schema_multi): + raise click.ClickException("--models cannot be used with schemas") if force_chat and prompt is not None: raise click.ClickException("--chat cannot be used with a prompt") + if schema_multi: + schema_input = schema_multi + schema = None + if schema_input: + # Never create logs.db for this unlogged command. An existing + # database can resolve stored schema IDs; all other schema input + # is resolved using a temporary in-memory database. + log_path = logs_db_path() + if log_path.exists(): + schema_db = sqlite_utils.Database(log_path) + else: + schema_db = sqlite_utils.Database(memory=True) + migrate(schema_db) + schema = resolve_schema_input(schema_db, schema_input, load_template) + if schema_multi: + schema = multi_schema(schema) + template_obj = None params = dict(param) if template: @@ -548,6 +584,8 @@ def endpoint( raise click.ClickException(str(ex)) if not model_id and template_obj.model: model_id = template_obj.model + if template_obj.schema_object: + schema = template_obj.schema_object if template_obj.options: options = _merge_template_options(template_obj, options) tools, python_tools = _merge_template_tools( @@ -572,6 +610,7 @@ def endpoint( "reasoning": True, "verbosity": True, "image_detail_original": True, + "supports_schema": True, "supports_tools": True, } if use_responses: @@ -595,6 +634,7 @@ def endpoint( prompt_kwargs = { "options": validated_options, + "schema": schema, "stream": not no_stream, "hide_reasoning": hide_reasoning, } @@ -661,6 +701,7 @@ def execute_chat_prompt(chat_prompt, _fragments, turn_attachments): elif ( prompt is None and not resolved_attachments + and not schema and (template_obj is None or "input" in template_obj.vars()) ): # Match `llm prompt`: wait for stdin until EOF instead of @@ -668,7 +709,7 @@ def execute_chat_prompt(chat_prompt, _fragments, turn_attachments): prompt = sys.stdin.read() if template_obj: prompt, system = _apply_template(template_obj, prompt, params, system) - if prompt is None: + if prompt is None and not (resolved_attachments or schema): raise click.ClickException( "A prompt is required when stdin is not interactive" ) diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py index 3c3c5c444..a7659c011 100644 --- a/tests/test_openai_endpoint.py +++ b/tests/test_openai_endpoint.py @@ -1,10 +1,12 @@ import base64 import json +import sqlite_utils from click.testing import CliRunner from pytest_httpx import IteratorStream from llm.cli import cli +from llm.migrations import migrate def _add_chat_response(httpx_mock, url, text): @@ -250,6 +252,13 @@ def test_endpoint_template(httpx_mock, user_path, templates_path): prompt: "Question: $input" options: temperature: 0.4 +schema_object: + type: object + properties: + answer: + type: string + required: + - answer attachment_types: - type: image/jpeg value: https://images.example.test/template.jpg @@ -266,6 +275,8 @@ def test_endpoint_template(httpx_mock, user_path, templates_path): "Where?", "--template", "endpoint", + "--schema", + '{"type": "object"}', "--param", "persona", "concise", @@ -294,11 +305,118 @@ def test_endpoint_template(httpx_mock, user_path, templates_path): }, ], "model": "template-model", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "output", + "schema": { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + }, + }, + }, "stream": False, "temperature": 0.4, } +def test_endpoint_schema(httpx_mock, user_path): + base_url = "https://schema.example.test/v1" + _add_chat_response(httpx_mock, base_url, '{"name": "Cleo", "age": 10}') + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Invent a dog", + "-m", + "test-model", + "--schema", + "name, age int", + "--no-stream", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert not (user_path / "logs.db").exists() + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["response_format"] == { + "type": "json_schema", + "json_schema": { + "name": "output", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + }, + "required": ["name", "age"], + }, + }, + } + + +def test_endpoint_schema_by_id_from_existing_logs_database(httpx_mock, user_path): + base_url = "https://schema-id.example.test/v1" + _add_chat_response(httpx_mock, base_url, '{"name": "Cleo"}') + schema = { + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + } + db = sqlite_utils.Database(str(user_path / "logs.db")) + migrate(db) + db["schemas"].insert({"id": "dog-schema", "content": json.dumps(schema)}) + assert (user_path / "logs.db").exists() + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Invent a dog", + "-m", + "test-model", + "--schema", + "dog-schema", + "--no-stream", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["response_format"]["json_schema"]["schema"] == schema + assert db["responses"].count == 0 + + +def test_endpoint_invalid_schema_id_does_not_create_logs_database(user_path): + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + "https://schema-id.example.test/v1", + "Invent a dog", + "-m", + "test-model", + "--schema", + "missing-schema", + "--no-stream", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 2 + assert "Invalid schema" in result.output + assert not (user_path / "logs.db").exists() + + def test_endpoint_static_template_runs_once_on_terminal( httpx_mock, user_path, templates_path, monkeypatch ): @@ -615,6 +733,59 @@ def test_endpoint_responses_api_attachment(httpx_mock, user_path): ] +def test_endpoint_responses_api_schema_multi(httpx_mock, user_path): + base_url = "https://responses-schema.example.test/v1" + httpx_mock.add_response( + method="POST", + url=f"{base_url}/responses", + json=_responses_payload('{"items": [{"name": "Cleo", "age": 10}]}'), + headers={"Content-Type": "application/json"}, + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Invent a dog", + "-m", + "test-model", + "--responses", + "--schema-multi", + "name, age int", + "--no-stream", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert not (user_path / "logs.db").exists() + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["text"]["format"] == { + "type": "json_schema", + "name": "output", + "schema": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + }, + "required": ["name", "age"], + }, + } + }, + "required": ["items"], + }, + "strict": False, + } + + def test_endpoint_responses_api_tools(httpx_mock, user_path): base_url = "https://responses-tools.example.test/v1" httpx_mock.add_response( From 46db435966289bacf12ef35b9ad40d9b3fc9cad7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 14:27:21 -0700 Subject: [PATCH 191/258] Promote uvx llm openai endpoint in README Refs #1565 --- README.md | 14 ++++++++++++++ docs/index.md | 12 ++++++++++++ 2 files changed, 26 insertions(+) diff --git a/README.md b/README.md index 172d8a174..16a8c9e20 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,20 @@ Or with [uv](https://docs.astral.sh/uv/guides/tools/) uv tool install llm ``` +Use LLM to run prompts or start chats against an arbitrary OpenAI-compatible Chat Completions endpoint, such as [LM Studio](https://lmstudio.ai). With `uvx`, you can do this without installing LLM first: + +```bash +uvx llm openai endpoint http://localhost:1234/v1 \ + -m google/gemma-4-12b \ + "What is the capital of France?" + +uvx llm openai endpoint http://localhost:1234/v1 \ + -m google/gemma-4-12b \ + --chat +``` + +Add `--key your-api-key` if the endpoint requires authentication. See [Run against an endpoint without configuring it](https://llm.datasette.io/en/stable/other-models.html#openai-endpoint) for more options. + If you have an [OpenAI API key](https://platform.openai.com/api-keys) key you can run this: ```bash diff --git a/docs/index.md b/docs/index.md index 8cfe293f0..62abeeb98 100644 --- a/docs/index.md +++ b/docs/index.md @@ -39,6 +39,18 @@ Or with [uv](https://docs.astral.sh/uv/guides/tools/) ```bash uv tool install llm ``` +Use LLM to run prompts or start chats against an arbitrary OpenAI-compatible Chat Completions endpoint, such as [LM Studio](https://lmstudio.ai). With `uvx`, you can do this without installing LLM first: +```bash +uvx llm openai endpoint http://localhost:1234/v1 \ + -m google/gemma-4-12b \ + "What is the capital of France?" + +uvx llm openai endpoint http://localhost:1234/v1 \ + -m google/gemma-4-12b \ + --chat +``` +Add `--key your-api-key` if the endpoint requires authentication. See {ref}`Run against an endpoint without configuring it ` for more options. + If you have an [OpenAI API key](https://platform.openai.com/api-keys) key you can run this: ```bash # Paste your OpenAI API key into this From 148c19df97ee639cdf473ac51cdc16cf7d08d255 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 15:01:41 -0700 Subject: [PATCH 192/258] Drop dependency on sqlite-migrate, closes #1577 --- docs/changelog.md | 1 + llm/embeddings_migrations.py | 2 +- mypy.ini | 3 --- pyproject.toml | 1 - 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 10f6337d8..37159b4cc 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -4,6 +4,7 @@ - The default model for users who have not set their own default is now [GPT-5.6 Luna](https://developers.openai.com/api/docs/models/gpt-5.6-luna). It was previously [GPT-4o mini](https://developers.openai.com/api/docs/models/gpt-4o-mini). Luna is a much better and more recent model, albeit slightly more expensive - $0.20 per million input tokens and $1.20 per million output tokens, compared to $0.15/$0.60 for 4o mini. You can switch back to 4o mini using `llm models default gpt-4o-mini`, or switch to [GPT-5 nano](https://developers.openai.com/api/docs/models/gpt-5-nano), an even cheaper default model ($0.05/$0.40), using `llm models default gpt-5-nano`. - New {ref}`llm openai endpoint ` command for running prompts, chats and model listings against arbitrary OpenAI-compatible endpoints without first configuring a model. These calls are not logged. [#1565](https://github.com/simonw/llm/issues/1565) +- No longer depends on [sqlite-migrate](https://github.com/simonw/sqlite-migrate), since that functionality is now handled by [sqlite-utils 4.0](https://sqlite-utils.datasette.io/en/stable/changelog.html#v4-0). [#1577](https://github.com/simonw/llm/issues/1577) (v0_32_rc1)= ## 0.32rc1 diff --git a/llm/embeddings_migrations.py b/llm/embeddings_migrations.py index eab9428c3..196d2abbb 100644 --- a/llm/embeddings_migrations.py +++ b/llm/embeddings_migrations.py @@ -1,7 +1,7 @@ import hashlib import time -from sqlite_migrate import Migrations +from sqlite_utils import Migrations embeddings_migrations = Migrations("llm.embeddings") diff --git a/mypy.ini b/mypy.ini index a17287e45..cfc209480 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,6 +5,3 @@ ignore_missing_imports = True [mypy-click_default_group.*] ignore_missing_imports = True - -[mypy-sqlite_migrate.*] -ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index 18cf6b572..b85a75770 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,6 @@ dependencies = [ "openai>=2.32.0", "click-default-group>=1.2.3", "sqlite-utils>=4.0", - "sqlite-migrate==0.1a2", "pydantic>=2.0.0", "PyYAML", "pluggy", From 293d8ecfe72bd8c094015fb42c1ffceec6f3e69f Mon Sep 17 00:00:00 2001 From: ikatyal2110 <134458944+ikatyal2110@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:12:11 -0500 Subject: [PATCH 193/258] Fix pending_tool_calls to exclude server_executed tool calls (#1575) LogStore.pending_tool_calls returned all ToolCallParts at the chain tip, including those with server_executed=True that the provider has already handled. This mirrors the same filter that _trailing_pending_tool_calls in models.py applies. --- llm/logs.py | 6 +++++- tests/test_logs_store.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/llm/logs.py b/llm/logs.py index 0e3f6e973..a0157cbbe 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -641,7 +641,11 @@ def pending_tool_calls(self, tip: str | None) -> list[Any]: chain = self.load_chain(tip) if not chain: return [] - return [part for part in chain[-1].parts if isinstance(part, ToolCallPart)] + return [ + part + for part in chain[-1].parts + if isinstance(part, ToolCallPart) and not part.server_executed + ] def ensure_attachment(db, attachment) -> str: diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index e68812424..e6fdf4899 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -475,6 +475,22 @@ def test_a_plain_reply_has_nothing_pending(self, store): tip = store.ensure_chain([llm.user("Hi"), llm.assistant("Hello")]) assert store.pending_tool_calls(tip) == [] + def test_server_executed_tool_calls_are_not_pending(self, store): + tip = store.ensure_chain( + [ + llm.user("Search"), + llm.assistant( + ToolCallPart( + name="web_search", + arguments={"q": "pelicans"}, + tool_call_id="tc1", + server_executed=True, + ) + ), + ] + ) + assert store.pending_tool_calls(tip) == [] + # ---- logging a response ---------------------------------------------- From 42e5a0f8c8a5af71fea852961045b97c96839188 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 15:02:29 -0700 Subject: [PATCH 194/258] Link new default model release notes to #1576 --- docs/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 37159b4cc..85751ca66 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,7 +2,7 @@ ## Unreleased -- The default model for users who have not set their own default is now [GPT-5.6 Luna](https://developers.openai.com/api/docs/models/gpt-5.6-luna). It was previously [GPT-4o mini](https://developers.openai.com/api/docs/models/gpt-4o-mini). Luna is a much better and more recent model, albeit slightly more expensive - $0.20 per million input tokens and $1.20 per million output tokens, compared to $0.15/$0.60 for 4o mini. You can switch back to 4o mini using `llm models default gpt-4o-mini`, or switch to [GPT-5 nano](https://developers.openai.com/api/docs/models/gpt-5-nano), an even cheaper default model ($0.05/$0.40), using `llm models default gpt-5-nano`. +- The default model for users who have not set their own default is now [GPT-5.6 Luna](https://developers.openai.com/api/docs/models/gpt-5.6-luna). It was previously [GPT-4o mini](https://developers.openai.com/api/docs/models/gpt-4o-mini). Luna is a much better and more recent model, albeit slightly more expensive - $0.20 per million input tokens and $1.20 per million output tokens, compared to $0.15/$0.60 for 4o mini. You can switch back to 4o mini using `llm models default gpt-4o-mini`, or switch to [GPT-5 nano](https://developers.openai.com/api/docs/models/gpt-5-nano), an even cheaper default model ($0.05/$0.40), using `llm models default gpt-5-nano`. [#1576](https://github.com/simonw/llm/issues/1576) - New {ref}`llm openai endpoint ` command for running prompts, chats and model listings against arbitrary OpenAI-compatible endpoints without first configuring a model. These calls are not logged. [#1565](https://github.com/simonw/llm/issues/1565) - No longer depends on [sqlite-migrate](https://github.com/simonw/sqlite-migrate), since that functionality is now handled by [sqlite-utils 4.0](https://sqlite-utils.datasette.io/en/stable/changelog.html#v4-0). [#1577](https://github.com/simonw/llm/issues/1577) From 4239f1742545e581b56f9d19799481481e5734e2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 15:15:23 -0700 Subject: [PATCH 195/258] Changelog entry for #1574 --- docs/changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.md b/docs/changelog.md index 85751ca66..407f363aa 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -5,6 +5,7 @@ - The default model for users who have not set their own default is now [GPT-5.6 Luna](https://developers.openai.com/api/docs/models/gpt-5.6-luna). It was previously [GPT-4o mini](https://developers.openai.com/api/docs/models/gpt-4o-mini). Luna is a much better and more recent model, albeit slightly more expensive - $0.20 per million input tokens and $1.20 per million output tokens, compared to $0.15/$0.60 for 4o mini. You can switch back to 4o mini using `llm models default gpt-4o-mini`, or switch to [GPT-5 nano](https://developers.openai.com/api/docs/models/gpt-5-nano), an even cheaper default model ($0.05/$0.40), using `llm models default gpt-5-nano`. [#1576](https://github.com/simonw/llm/issues/1576) - New {ref}`llm openai endpoint ` command for running prompts, chats and model listings against arbitrary OpenAI-compatible endpoints without first configuring a model. These calls are not logged. [#1565](https://github.com/simonw/llm/issues/1565) - No longer depends on [sqlite-migrate](https://github.com/simonw/sqlite-migrate), since that functionality is now handled by [sqlite-utils 4.0](https://sqlite-utils.datasette.io/en/stable/changelog.html#v4-0). [#1577](https://github.com/simonw/llm/issues/1577) +- Fixed a bug where server-executed tool calls were incorrectly reported as pending by `LogStore.pending_tool_calls()`. Thanks, [ikatyal2110](https://github.com/ikatyal2110). [#1574](https://github.com/simonw/llm/issues/1574) (v0_32_rc1)= ## 0.32rc1 From 8dfd6b4a48ea0f2a3d17834d87de1b8e66daba9f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 15:33:02 -0700 Subject: [PATCH 196/258] Better error from schema_dsl() for malformed fields Closes #1466, #1467, #1469, #1487, #1544 --- docs/changelog.md | 1 + llm/utils.py | 4 ++++ tests/test_utils.py | 15 +++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 407f363aa..c81686f5d 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -6,6 +6,7 @@ - New {ref}`llm openai endpoint ` command for running prompts, chats and model listings against arbitrary OpenAI-compatible endpoints without first configuring a model. These calls are not logged. [#1565](https://github.com/simonw/llm/issues/1565) - No longer depends on [sqlite-migrate](https://github.com/simonw/sqlite-migrate), since that functionality is now handled by [sqlite-utils 4.0](https://sqlite-utils.datasette.io/en/stable/changelog.html#v4-0). [#1577](https://github.com/simonw/llm/issues/1577) - Fixed a bug where server-executed tool calls were incorrectly reported as pending by `LogStore.pending_tool_calls()`. Thanks, [ikatyal2110](https://github.com/ikatyal2110). [#1574](https://github.com/simonw/llm/issues/1574) +- `schema_dsl()` now raises a descriptive `ValueError` instead of an `IndexError` for malformed fields with no name before the colon. [#1466](https://github.com/simonw/llm/issues/1466) (v0_32_rc1)= ## 0.32rc1 diff --git a/llm/utils.py b/llm/utils.py index 4bf48d50d..5f6cfddb7 100644 --- a/llm/utils.py +++ b/llm/utils.py @@ -391,6 +391,10 @@ def schema_dsl(schema_dsl: str, multi: bool = False) -> dict[str, Any]: # Process field name and type field_parts = field_info.strip().split() + if not field_parts: + raise ValueError( + f"Invalid schema DSL: field {field!r} is missing a name before ':'" + ) field_name = field_parts[0].strip() # Default type is string diff --git a/tests/test_utils.py b/tests/test_utils.py index aa9c8093c..6b8061258 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -254,6 +254,21 @@ def test_schema_dsl_multi(): } +@pytest.mark.parametrize( + ("schema", "invalid_field"), + ( + (":just a description", ":just a description"), + ("name, :description", ":description"), + ), +) +def test_schema_dsl_missing_field_name(schema, invalid_field): + with pytest.raises(ValueError) as ex: + schema_dsl(schema) + assert str(ex.value) == ( + f"Invalid schema DSL: field {invalid_field!r} is missing a name before ':'" + ) + + @pytest.mark.parametrize( "text, max_length, normalize_whitespace, keep_end, expected", [ From 9efdfa6d5836011e4d9c7071a500e0e503368164 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 15:43:54 -0700 Subject: [PATCH 197/258] Fix fragment filtering on SQLite 3.51 SQLite 3.51.0 and 3.51.1 can return incorrect results for EXISTS around a UNION. Use separate EXISTS clauses joined by OR so prompt and system fragment matches are both preserved. Based on #1571 by @ikatyal2110. Closes #1511 --- llm/logs.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/llm/logs.py b/llm/logs.py index a0157cbbe..65592e759 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -1380,17 +1380,20 @@ def legacy_log_rows( for index, fragment_hash in enumerate(fragment_hashes): key = f"fragment_{index}" - where.append(f"""exists ( - select 1 from prompt_fragments - where prompt_fragments.response_id = responses.id - and prompt_fragments.fragment_id in ( - select fragments.id from fragments where hash = :{key} + where.append(f"""( + exists ( + select 1 from prompt_fragments + where prompt_fragments.response_id = responses.id + and prompt_fragments.fragment_id in ( + select fragments.id from fragments where hash = :{key} + ) ) - union - select 1 from system_fragments - where system_fragments.response_id = responses.id - and system_fragments.fragment_id in ( - select fragments.id from fragments where hash = :{key} + or exists ( + select 1 from system_fragments + where system_fragments.response_id = responses.id + and system_fragments.fragment_id in ( + select fragments.id from fragments where hash = :{key} + ) ) )""") params[key] = fragment_hash From 797a2652e3ed97df7525d54f6b69f75777080f14 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 15:49:56 -0700 Subject: [PATCH 198/258] Release 0.32rc2 Refs #1466, #1511, #1565, #1568, #1574, #1575, #1576, #1577 --- docs/changelog.md | 3 ++- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index c81686f5d..4dddf81d0 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,6 +1,7 @@ # Changelog -## Unreleased +(v0_32_rc2)= +## 0.32rc2 - The default model for users who have not set their own default is now [GPT-5.6 Luna](https://developers.openai.com/api/docs/models/gpt-5.6-luna). It was previously [GPT-4o mini](https://developers.openai.com/api/docs/models/gpt-4o-mini). Luna is a much better and more recent model, albeit slightly more expensive - $0.20 per million input tokens and $1.20 per million output tokens, compared to $0.15/$0.60 for 4o mini. You can switch back to 4o mini using `llm models default gpt-4o-mini`, or switch to [GPT-5 nano](https://developers.openai.com/api/docs/models/gpt-5-nano), an even cheaper default model ($0.05/$0.40), using `llm models default gpt-5-nano`. [#1576](https://github.com/simonw/llm/issues/1576) - New {ref}`llm openai endpoint ` command for running prompts, chats and model listings against arbitrary OpenAI-compatible endpoints without first configuring a model. These calls are not logged. [#1565](https://github.com/simonw/llm/issues/1565) diff --git a/pyproject.toml b/pyproject.toml index b85a75770..671c8f827 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm" -version = "0.32rc1" +version = "0.32rc2" description = "CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine." readme = { file = "README.md", content-type = "text/markdown" } authors = [ From e9ec05995684ea820062ff80a443509b65437cdd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jul 2026 15:51:56 -0700 Subject: [PATCH 199/258] Cog for version number --- docs/fragments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/fragments.md b/docs/fragments.md index 5734a333e..2f1e0df90 100644 --- a/docs/fragments.md +++ b/docs/fragments.md @@ -30,7 +30,7 @@ from importlib.metadata import version llm_version = version("llm") cog.out(f'The URL will be fetched with the user-agent `llm/{llm_version} (https://llm.datasette.io/)`.') ]]]--> -The URL will be fetched with the user-agent `llm/0.32rc1 (https://llm.datasette.io/)`. +The URL will be fetched with the user-agent `llm/0.32rc2 (https://llm.datasette.io/)`. The `-f` option can be used multiple times to combine together multiple fragments. From b92f4bcc6e68ca367d125412a13f9487d08179a1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 1 Aug 2026 17:52:13 -0700 Subject: [PATCH 200/258] Dates for 0.32rc1 and 0.32rc2 --- docs/changelog.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 4dddf81d0..4b31853ba 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,7 +1,7 @@ # Changelog (v0_32_rc2)= -## 0.32rc2 +## 0.32rc2 (2026-07-30) - The default model for users who have not set their own default is now [GPT-5.6 Luna](https://developers.openai.com/api/docs/models/gpt-5.6-luna). It was previously [GPT-4o mini](https://developers.openai.com/api/docs/models/gpt-4o-mini). Luna is a much better and more recent model, albeit slightly more expensive - $0.20 per million input tokens and $1.20 per million output tokens, compared to $0.15/$0.60 for 4o mini. You can switch back to 4o mini using `llm models default gpt-4o-mini`, or switch to [GPT-5 nano](https://developers.openai.com/api/docs/models/gpt-5-nano), an even cheaper default model ($0.05/$0.40), using `llm models default gpt-5-nano`. [#1576](https://github.com/simonw/llm/issues/1576) - New {ref}`llm openai endpoint ` command for running prompts, chats and model listings against arbitrary OpenAI-compatible endpoints without first configuring a model. These calls are not logged. [#1565](https://github.com/simonw/llm/issues/1565) @@ -10,7 +10,7 @@ - `schema_dsl()` now raises a descriptive `ValueError` instead of an `IndexError` for malformed fields with no name before the colon. [#1466](https://github.com/simonw/llm/issues/1466) (v0_32_rc1)= -## 0.32rc1 +## 0.32rc1 (2026-07-30) This release candidate for 0.32 introduces a new database schema for logging prompts and responses that captures full details of the interaction with the underlying LLM, and de-duplicates those records using a **content-addressed message store**. From 11374cb4146529a6ab4da297036e011cf62ef60b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 2 Aug 2026 15:50:09 -0700 Subject: [PATCH 201/258] Store response.json() on turns, condensed with condense-json The raw provider payload was dropped entirely when logging moved to the message store - the chain became the record, at the cost of everything with no part equivalent: usage breakdowns, fingerprints, logprobs, settings echoes. This restores it as a turns.response_json column without re-duplicating what the parts tables already hold. The payload is stored condensed: strings of 64+ characters that also appear in the turn's own messages (response text, reasoning summaries and their encrypted blobs, long tool arguments, tool output) are replaced with {"$": "0.0.text"}-style references keyed by message offset, part position and field path. The replacement dict is never stored - it is rebuilt from the chain segment between the turn's parent and tip on the way out, which is sound because message content is hash-frozen. What remains is roughly the provider envelope; for a typical text response the column costs a few hundred bytes regardless of output length. condense-json 1.0 (dependency bumped from 0.1.3) is what makes this safe on arbitrary payloads: marker-shaped input is escaped via $raw for a guaranteed lossless round trip, and unknown references raise a typed UncondenseError. Reading: llm logs --json now carries response_json for new-store rows, resolved back to exactly what the provider sent (legacy rows already showed theirs); LogStore.turn_response_json() is the Python API. A payload whose references no longer resolve renders as absent in the listing and raises UncondenseError from the API. The streamed Responses API path needed one fix to dedupe: reasoning metadata was harvested from response.output_item.done while response_json came from response.completed, and OpenAI encrypts per event, so the part and the payload held different ciphertexts of the same reasoning. The plugin now re-emits reasoning provider_metadata from the final payload after the stream ends, aimed at the part_index the framework resolved onto the retained done-event, so both records agree on one blob - measured savings on streamed gpt-5-mini reasoning go from 15% to 50%. Not covered, deliberately: prompt_json stays unrecorded, and streamed chat-completions payloads still lack tool_calls (combine_chunks never carried them) - the parts remain the richer record on that path. Co-Authored-By: Claude Fable 5 --- docs/logging.md | 15 ++- llm/default_plugins/openai_models.py | 63 ++++++++++++- llm/logs.py | 134 ++++++++++++++++++++++++++- llm/migrations.py | 12 +++ pyproject.toml | 2 +- tests/test_logs_store.py | 129 ++++++++++++++++++++++++++ tests/test_openai_responses.py | 79 ++++++++++++++++ 7 files changed, 424 insertions(+), 10 deletions(-) diff --git a/docs/logging.md b/docs/logging.md index 338986ac5..907393655 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -530,6 +530,16 @@ Here fragment `1` is an id in the existing `fragments` table. Reading the part c Attachments work the same way: the binary content lives in the `attachments` table, keyed by a SHA-256 hash of the bytes, and the part payload stores that id in place of the data. +(logging-message-store-response-json)= + +### The raw provider payload + +The parts of a response are a normalized view of what the provider returned - enough to replay the conversation, but not the whole story. The raw `response.json()` dictionary is also recorded on the turn, in the `turns.response_json` column, so the details that have no part equivalent - usage breakdowns, system fingerprints, logprobs, settings echoes - survive too. + +It is stored *condensed*, using [condense-json](https://github.com/simonw/condense-json). Any string of 64 characters or more that also appears in the turn's own messages - the response text, a reasoning summary and its encrypted counterpart, long tool arguments - is replaced with a reference such as `{"$": "0.0.text"}`, keyed by the message offset, part position and field the string came from. The replacement mapping is never stored: it is rebuilt from the stored messages whenever the payload is read, which is sound because message content is frozen by its hash. What remains in the column is roughly the provider envelope, at a fraction of the size of the raw payload and without a second copy of anything the parts already hold. + +`llm logs --json` resolves the stored payload back to exactly what the provider sent and includes it as `response_json`. From Python, `LogStore.turn_response_json(turn_id)` returns the same resolved dictionary. Turns logged before this column existed, and models that expose no raw payload, record NULL. + (logging-message-store-tables)= ### Table by table @@ -540,7 +550,7 @@ The full schema for these tables appears in {ref}`the SQL schema section ` in `response_json`. Turn ids are ULIDs, in the same id space as legacy response ids, which is how the two generations of tables sort together in `llm logs`. - `turn_tools` - which {ref}`tool ` definitions were available to a turn, referencing the `tools` table. For toolbox-derived tools, `instance_id` references the `tool_instances` row recording which configured instance provided them - so the tools list in `llm logs` shows that `SQLite_query` came from `SQLite("mydb.db")` before any call has run. - `turn_fragments` - which fragments a turn was given, with their `kind` (`prompt` or `system`) and order. Provenance lives here rather than on the shared message rows, and this table is what powers `llm logs -f`. - `turn_search` - the searchable text of each turn: the literal prompt the user typed (fragment content excluded, via the `text_ref` literals described above) and the assistant's text output. An FTS5 index over this table, `turn_search_fts`, is what powers {ref}`llm logs -q `. Derived from the stored parts when a turn is logged; a turn with no prompt or response text, such as a pure tool call, gets no row. @@ -869,7 +879,8 @@ CREATE TABLE "turns" ( "output_tokens" INTEGER, "token_details" TEXT, "duration_ms" INTEGER, - "datetime_utc" TEXT + "datetime_utc" TEXT, + "response_json" TEXT ); CREATE TABLE "turn_tools" ( "turn_id" TEXT REFERENCES "turns"("id"), diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index df4ebf60b..1a5770ba2 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1752,6 +1752,44 @@ def _reasoning_event(self, item, *, include_text=True): provider_metadata={"openai": meta} if meta else None, ) + def _reasoning_refresh_events(self, response_json, done_events): + """Metadata-only reasoning events rebuilt from the final payload. + + While streaming, reasoning metadata is first harvested from the + ``response.output_item.done`` event, but the ``response.completed`` + payload carries a *different* ciphertext of the same reasoning - + OpenAI encrypts per event. Re-emitting the metadata from the + final payload, aimed at the already-resolved part_index, makes + the stored part and ``response_json`` agree on one blob (which + also lets the log store condense the payload against the part). + + ``done_events`` maps reasoning item id to the StreamEvent + yielded at ``output_item.done``; the framework has resolved + ``part_index`` on it by the time the stream ends. + """ + events = [] + for item in response_json.get("output") or []: + if not isinstance(item, dict) or item.get("type") != "reasoning": + continue + prior = done_events.get(item.get("id")) + if prior is None or prior.part_index is None: + continue + meta = { + key: item[key] + for key in ("id", "encrypted_content", "summary") + if item.get(key) + } + if meta: + events.append( + StreamEvent( + type="reasoning", + chunk="", + part_index=prior.part_index, + provider_metadata={"openai": meta}, + ) + ) + return events + class Responses(_SharedResponses, KeyModel): needs_key = "openai" @@ -1855,6 +1893,7 @@ def execute( tool_call_meta: dict[str, dict[str, str]] = {} final_response_dict: dict[str, Any] | None = None reasoning_items_with_streamed_text = set() + reasoning_done_events: dict[str, StreamEvent] = {} for event in stream_obj: etype = getattr(event, "type", None) if etype == "response.output_item.added": @@ -1905,12 +1944,18 @@ def execute( if item.type == "reasoning": had_reasoning = True item_id = getattr(item, "id", None) - yield self._reasoning_event( + reasoning_event = self._reasoning_event( item, include_text=( item_id not in reasoning_items_with_streamed_text ), ) + if item_id: + # Retained so the refresh after + # response.completed can target the part + # this event resolved to. + reasoning_done_events[item_id] = reasoning_event + yield reasoning_event elif item.type == "function_call": try: args = json.loads(item.arguments) if item.arguments else {} @@ -1929,6 +1974,9 @@ def execute( usage = final_response_dict["usage"] if final_response_dict is not None: response.response_json = remove_dict_none_values(final_response_dict) + yield from self._reasoning_refresh_events( + response.response_json, reasoning_done_events + ) else: completion = client.responses.create( model=self.model_name or self.model_id, @@ -2089,6 +2137,7 @@ async def execute( tool_call_meta: dict[str, dict[str, str]] = {} final_response_dict: dict[str, Any] | None = None reasoning_items_with_streamed_text = set() + reasoning_done_events: dict[str, StreamEvent] = {} async for event in stream_obj: etype = getattr(event, "type", None) if etype == "response.output_item.added": @@ -2139,12 +2188,18 @@ async def execute( if item.type == "reasoning": had_reasoning = True item_id = getattr(item, "id", None) - yield self._reasoning_event( + reasoning_event = self._reasoning_event( item, include_text=( item_id not in reasoning_items_with_streamed_text ), ) + if item_id: + # Retained so the refresh after + # response.completed can target the part + # this event resolved to. + reasoning_done_events[item_id] = reasoning_event + yield reasoning_event elif item.type == "function_call": try: args = json.loads(item.arguments) if item.arguments else {} @@ -2163,6 +2218,10 @@ async def execute( usage = final_response_dict["usage"] if final_response_dict is not None: response.response_json = remove_dict_none_values(final_response_dict) + for refresh in self._reasoning_refresh_events( + response.response_json, reasoning_done_events + ): + yield refresh else: completion = await client.responses.create( model=self.model_name or self.model_id, diff --git a/llm/logs.py b/llm/logs.py index 65592e759..f5ad64309 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -21,6 +21,8 @@ import json from typing import Any +from condense_json import UncondenseError, condense_json, uncondense_json + from .migrations import migrate from .models import Attachment, _conversation_name from .parts import ( @@ -481,7 +483,8 @@ def _log_in_transaction(self, response, thread_id: str | None) -> str: ) # _messages_now() rather than messages(), which is a coroutine on # AsyncResponse. - tip = self.ensure_chain(response._messages_now(), parent=parent) + own_messages = response._messages_now() + tip = self.ensure_chain(own_messages, parent=parent) schema_id = None if response.prompt.schema: @@ -512,6 +515,9 @@ def _log_in_transaction(self, response, thread_id: str | None) -> str: "token_details": _dump(response.token_details), "duration_ms": response.duration_ms(), "datetime_utc": response.datetime_utc(), + "response_json": condense_payload( + getattr(response, "response_json", None), own_messages + ), }, replace=True, ) @@ -596,6 +602,30 @@ def _log_in_transaction(self, response, thread_id: str | None) -> str: self.db["threads"].update(thread_id, {"tip_message_hash": tip}) return turn_id + def turn_response_json(self, turn_id: str) -> Any: + """The raw provider payload recorded for a turn, resolved. + + Returns ``None`` when the turn is unknown or recorded no + payload. Raises ``condense_json.UncondenseError`` when the + payload references message content that no longer resolves - + the payload was recorded but its context is gone. + """ + row = next( + iter( + self.db.query( + "select parent_message_hash, tip_message_hash, response_json" + " from turns where id = ?", + [turn_id], + ) + ), + None, + ) + if row is None or row["response_json"] is None: + return None + inputs = self.load_chain(row["parent_message_hash"]) + outputs = self.load_chain(row["tip_message_hash"])[len(inputs) :] + return resolve_payload(row["response_json"], outputs) + # -- verification -------------------------------------------------- def verify(self) -> list[str]: @@ -789,6 +819,87 @@ def _now() -> str: return str(datetime.datetime.now(datetime.timezone.utc)) +# -- condensed provider payloads ---------------------------------------- +# +# The raw response.json() payload mostly duplicates content the message +# tables already hold: the response text, reasoning summaries and their +# encrypted blobs, long tool arguments. The turn stores it condensed +# instead - strings that already live in the turn's own messages are +# swapped for {"$": key} references (condense-json), leaving roughly the +# provider envelope: ids, usage, fingerprints, settings echoes. The +# replacement dict is never stored; it is rebuilt from the chain segment +# on the way out, which is sound because message content is hash-frozen, +# so the same walk over the same messages produces the same dict on both +# sides of the round trip. + +# Below this a {"$": key} marker costs about as much as the string it +# replaces. +_CONDENSE_MIN_LENGTH = 64 + + +def _payload_replacements(messages) -> dict[str, str]: + """Replacement strings for condensing a turn's provider payload. + + ``messages`` is the turn's own contribution - the chain segment + between its parent and tip. Keys are structural (message offset, + part position, field path) so the identical dict can be rebuilt + from the stored segment at read time. + """ + replacements: dict[str, str] = {} + + def add(key: str, value: Any) -> None: + if isinstance(value, str) and len(value) >= _CONDENSE_MIN_LENGTH: + replacements[key] = value + + def walk(prefix: str, obj: Any) -> None: + if isinstance(obj, dict): + for key, value in obj.items(): + walk(f"{prefix}.{key}", value) + elif isinstance(obj, list): + for index, value in enumerate(obj): + walk(f"{prefix}.{index}", value) + else: + add(prefix, obj) + + for mi, message in enumerate(messages): + message_dict = message.to_dict() + for pi, part in enumerate(message_dict.get("parts", [])): + base = f"{mi}.{pi}" + add(f"{base}.text", part.get("text")) + add(f"{base}.output", part.get("output")) + arguments = part.get("arguments") + if arguments: + # Providers carry tool arguments as a JSON-encoded + # string; OpenAI uses the compact form, so offer both + # serializations of the stored dict. + compact = json.dumps(arguments, separators=(",", ":")) + spaced = json.dumps(arguments) + add(f"{base}.args", compact) + if spaced != compact: + add(f"{base}.args2", spaced) + walk(f"{base}.pm", part.get("provider_metadata") or {}) + walk(f"{mi}.pm", message_dict.get("provider_metadata") or {}) + return replacements + + +def condense_payload(payload: Any, messages) -> str | None: + "JSON text of ``payload`` with strings from ``messages`` condensed." + if payload is None: + return None + return json.dumps(condense_json(payload, _payload_replacements(messages))) + + +def resolve_payload(condensed: str | None, messages) -> Any: + """Reverse of :func:`condense_payload`. + + Raises ``condense_json.UncondenseError`` when the stored payload + references content the segment no longer produces. + """ + if condensed is None: + return None + return uncondense_json(json.loads(condensed), _payload_replacements(messages)) + + # -- llm logs support --------------------------------------------------- # # Rows shaped like the ones the older `responses` query produced, so the @@ -809,6 +920,7 @@ def _now() -> str: turns.token_details, turns.parent_message_hash, turns.tip_message_hash, + turns.response_json, threads.name as conversation_name, turns.model as conversation_model, schemas.content as schema_json{rank_select} @@ -963,11 +1075,14 @@ def build(self, row: dict) -> dict: "system": _text_of(system_parts, TextPart) or None, "response": _text_of(out_parts, TextPart), "reasoning": _text_of(out_parts, ReasoningPart) or None, - # Neither is stored any more: the chain holds the - # structure, and the raw provider payload was dropped as - # redundant with it. + # No longer stored: the chain holds the structure. "prompt_json": None, - "response_json": None, + # Stored condensed against the turn's own messages; + # resolved here so the row carries the payload as the + # provider sent it. A payload whose references no longer + # resolve renders as absent rather than failing the + # whole listing. + "response_json": self._resolve_response_json(row, outputs), "_input_parts": input_parts, "_output_parts": out_parts, # Internal, stripped before rendering - the enrichment @@ -981,6 +1096,15 @@ def build(self, row: dict) -> dict: ) return built + def _resolve_response_json(self, row: dict, outputs) -> str | None: + condensed = row.get("response_json") + if not condensed: + return None + try: + return json.dumps(resolve_payload(condensed, outputs)) + except UncondenseError: + return None + def _input_segment_hashes(self, parent_hash: str | None) -> list[str]: """Hashes of the turn's own input messages - the same trailing user/tool run build() derives, walked directly in the table.""" diff --git a/llm/migrations.py b/llm/migrations.py index 73d282c0d..bc26a5759 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -720,3 +720,15 @@ def m026_message_tree_view(db): # path in the output - selecting from the view preserves tree order # only while sorted by path. db.create_view("message_tree", MESSAGE_TREE_SQL) + + +@migration +def m027_turns_response_json(db): + # The raw provider payload for the call, stored condensed: strings + # that already live in the turn's message parts (response text, + # reasoning blobs, long tool arguments) are replaced with references + # via condense-json, so the column costs roughly the provider + # envelope - ids, usage, fingerprints - not a second copy of the + # response. NULL for turns logged before this column existed and + # for models that expose no raw payload. + db["turns"].add_column("response_json", str) diff --git a/pyproject.toml b/pyproject.toml index 671c8f827..37cb32ee1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ dependencies = [ "click", - "condense-json>=0.1.3", + "condense-json>=1.0", "openai>=2.32.0", "click-default-group>=1.2.3", "sqlite-utils>=4.0", diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index e6fdf4899..b9ca29c23 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -1719,3 +1719,132 @@ def test_messages_no_turn_has_recorded_have_no_datetime(self, store): store.append(store.create_thread(), [llm.user("Hi")]) (row,) = self.rows(store) assert row["datetime"] is None + + +# ---- condensed provider payloads ------------------------------------- + + +class TestResponseJsonPayload: + """The raw response.json() payload is stored on the turn, condensed + against the strings the turn's own messages already hold, and + resolved again on the way out.""" + + LONG = "SQLite stores the whole database in a single file on disk. " * 3 + + def logged(self, store, mock_model, payload, text=None): + mock_model.enqueue([text if text is not None else self.LONG]) + response = mock_model.prompt("Tell me about SQLite") + response.text() + response.response_json = payload + return store.log(response) + + def test_payload_is_stored_condensed(self, store, mock_model): + turn_id = self.logged( + store, + mock_model, + {"content": self.LONG, "id": "chatcmpl-1", "usage": {"total_tokens": 9}}, + ) + stored = store.db["turns"].get(turn_id)["response_json"] + # The response text is a reference, not a second copy + assert json.loads(stored)["content"] == {"$": "0.0.text"} + assert self.LONG not in stored + + def test_turn_response_json_resolves_the_payload(self, store, mock_model): + payload = {"content": self.LONG, "id": "chatcmpl-1"} + turn_id = self.logged(store, mock_model, payload) + assert store.turn_response_json(turn_id) == payload + + def test_no_payload_stores_null(self, store, mock_model): + turn_id = self.logged(store, mock_model, None) + assert store.db["turns"].get(turn_id)["response_json"] is None + assert store.turn_response_json(turn_id) is None + + def test_unknown_turn_resolves_to_none(self, store): + assert store.turn_response_json("nope") is None + + def test_short_strings_are_stored_verbatim(self, store, mock_model): + # Below the length threshold a reference would cost as much as + # the string it replaces. + turn_id = self.logged( + store, mock_model, {"content": "short answer"}, text="short answer" + ) + stored = store.db["turns"].get(turn_id)["response_json"] + assert json.loads(stored) == {"content": "short answer"} + + def test_marker_shaped_payload_round_trips(self, store, mock_model): + # A payload that already contains {"$": ...} shapes must come + # back exactly - condense-json escapes them with $raw. + payload = {"tricky": {"$": "not-a-marker"}, "also": {"$r": ["x"]}} + turn_id = self.logged(store, mock_model, payload) + assert store.turn_response_json(turn_id) == payload + + def test_merged_log_rows_carry_the_resolved_payload(self, store, mock_model): + payload = {"content": self.LONG, "id": "chatcmpl-1"} + self.logged(store, mock_model, payload) + (row,) = merged_log_rows(store) + assert json.loads(row["response_json"]) == payload + + def test_a_payload_that_no_longer_resolves_is_absent_from_rows( + self, store, mock_model + ): + turn_id = self.logged(store, mock_model, {"content": self.LONG}) + store.db["turns"].update( + turn_id, {"response_json": json.dumps({"content": {"$": "9.9.text"}})} + ) + (row,) = merged_log_rows(store) + assert row["response_json"] is None + # The API surfaces the failure instead of guessing + from condense_json import UncondenseError + + with pytest.raises(UncondenseError): + store.turn_response_json(turn_id) + + +class TestPayloadReplacements: + """_payload_replacements builds the same dict at write and read time + from the turn's chain segment - these pin down what it offers.""" + + def test_part_strings_are_keyed_by_position(self): + from llm.logs import _payload_replacements + + blob = "b" * 80 + messages = [ + llm.assistant( + llm.parts.ReasoningPart( + text="r" * 70, + provider_metadata={"openai": {"encrypted_content": blob}}, + ), + "t" * 64, + ) + ] + replacements = _payload_replacements(messages) + assert replacements == { + "0.0.text": "r" * 70, + "0.0.pm.openai.encrypted_content": blob, + "0.1.text": "t" * 64, + } + + def test_tool_arguments_offer_both_serializations(self): + from llm.logs import _payload_replacements + + arguments = {"query": "x" * 64} + messages = [ + llm.assistant(llm.parts.ToolCallPart(name="search", arguments=arguments)) + ] + replacements = _payload_replacements(messages) + assert replacements["0.0.args"] == json.dumps(arguments, separators=(",", ":")) + assert replacements["0.0.args2"] == json.dumps(arguments) + + def test_tool_result_output_is_offered(self): + from llm.logs import _payload_replacements + + output = "o" * 64 + messages = [ + llm.tool_message(llm.parts.ToolResultPart(name="search", output=output)) + ] + assert _payload_replacements(messages) == {"0.0.output": output} + + def test_short_strings_are_excluded(self): + from llm.logs import _payload_replacements + + assert _payload_replacements([llm.assistant("short")]) == {} diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index ebca2a6dc..f7e9c6588 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -676,3 +676,82 @@ def db_lookup(key: str) -> str: assert pm.get( "encrypted_content" ), "ReasoningPart missing encrypted_content" + + +def _responses_reasoning_refresh_stream(): + """Reasoning stream whose final payload carries a different ciphertext. + + OpenAI encrypts reasoning per event, so ``output_item.done`` and the + ``response.completed`` payload hold different encrypted_content for + the same reasoning item. + """ + yield from _responses_reasoning_summary_stream() + yield _responses_sse( + "response.completed", + { + "response": { + "id": "resp_1", + "object": "response", + "created_at": 1700000000, + "model": "gpt-5.5", + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "output": [ + { + "id": "rs_1", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Thinking aloud"}], + "encrypted_content": "encrypted-final", + "status": "completed", + }, + { + "id": "msg_1", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [ + {"type": "output_text", "text": "done", "annotations": []} + ], + }, + ], + "usage": { + "input_tokens": 1, + "output_tokens": 2, + "total_tokens": 3, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 1}, + }, + }, + "sequence_number": 5, + }, + ) + + +def test_responses_reasoning_metadata_refreshed_from_final_payload(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + stream=IteratorStream(_responses_reasoning_refresh_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + + model = llm.get_model("gpt-5.5") + response = model.prompt("hello", key="test") + assert response.text() == "done" + + reasoning_parts = [ + p + for m in response.messages() + for p in m.parts + if isinstance(p, llm.parts.ReasoningPart) + ] + # One reasoning part - the refresh merged into it rather than + # growing a second one - and it carries the final payload's + # ciphertext, the same string response_json holds. + assert len(reasoning_parts) == 1 + metadata = reasoning_parts[0].provider_metadata["openai"] + assert metadata["encrypted_content"] == "encrypted-final" + payload_item = response.response_json["output"][0] + assert payload_item["encrypted_content"] == "encrypted-final" + assert reasoning_parts[0].text == "Thinking aloud" From 72da1dfc430678a3b1f39fcba2d78f75a30730fa Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 2 Aug 2026 23:10:10 +0000 Subject: [PATCH 202/258] Ran cog --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 16a8c9e20..ec6f904ac 100644 --- a/README.md +++ b/README.md @@ -355,6 +355,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Content addressing as a contract](https://llm.datasette.io/en/stable/logging.html#content-addressing-as-a-contract) * [Forking and shared history](https://llm.datasette.io/en/stable/logging.html#forking-and-shared-history) * [Storage by reference](https://llm.datasette.io/en/stable/logging.html#storage-by-reference) + * [The raw provider payload](https://llm.datasette.io/en/stable/logging.html#the-raw-provider-payload) * [Table by table](https://llm.datasette.io/en/stable/logging.html#table-by-table) * [Querying the message store](https://llm.datasette.io/en/stable/logging.html#querying-the-message-store) * [Logging from Python](https://llm.datasette.io/en/stable/logging.html#logging-from-python) From bd645da55193f114b9e032e398d7730c92b1719a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 2 Aug 2026 16:14:06 -0700 Subject: [PATCH 203/258] llm tools now displays dynamic toolboxes usefully, closes #1580 --- docs/changelog.md | 5 ++ docs/python-api.md | 2 + docs/usage.md | 34 +++++++++ llm/cli.py | 85 ++++++++++++++++++----- llm/models.py | 2 + tests/test_plugins.py | 156 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 265 insertions(+), 19 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 4b31853ba..b5ff986f5 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,10 @@ # Changelog +(unreleased)= +## Unreleased + +- `llm tools` now provides improved output for dynamic toolboxes - toolbox classes such as `MCP` from [llm-mcp-client](https://github.com/simonw/llm-mcp-client) that generate their tools at runtime. These were previously listed as just their name. They are now listed with their constructor signature and class docstring. Passing one or more specifications such as `llm tools 'MCP("https://datasette.simonwillison.net/-/mcp")'` instantiates each toolbox and lists the tools that configured instance provides. `llm tools --json` output now includes a `"dynamic"` boolean key for each toolbox. [#1580](https://github.com/simonw/llm/issues/1580) + (v0_32_rc2)= ## 0.32rc2 (2026-07-30) diff --git a/docs/python-api.md b/docs/python-api.md index 26a237598..88a614a4e 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -401,6 +401,8 @@ In asynchronous contexts the alternative method `await toolbox.prepare_async()` If you want to prepare the class in this way such that it can be used in both synchronous and asynchronous contexts, implement both `prepare()` and `prepare_async()` methods. +Toolbox classes that override `tools()`, `prepare()` or `prepare_async()` are treated as dynamic by the `llm tools` command. Since their full list of tools cannot be determined until an instance has been configured, `llm tools` displays their constructor signature and class docstring instead of a list of tools - so give your dynamic toolbox a descriptive docstring that shows how to configure it. Users can then pass a full specification such as `llm tools 'MCP("https://example.com/mcp")'` to instantiate the toolbox - running `prepare()` if it is implemented - and list the tools that instance provides. See {ref}`the tools usage documentation ` for details. + (python-api-schemas)= ### Schemas diff --git a/docs/usage.md b/docs/usage.md index 00461354f..72af52963 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -207,6 +207,40 @@ Toolboxes always start with a capital letter. They can be configured by passing - Single JSON value: `ToolboxName("hello")` or `ToolboxName([1,2,3])` - Key-value pairs: `ToolboxName(name="test", count=5, items=[1,2])` - treated the same as `{"name": "test", "count": 5, "items": [1, 2]}`, all values must be valid JSON +The `llm tools` command lists toolboxes along with the tools they provide. Some toolboxes cannot know their list of tools until they have been configured - [llm-mcp-client](https://github.com/simonw/llm-mcp-client) for example fetches its tools from whichever MCP server it is pointed at. These dynamic toolboxes are listed with their constructor signature and documentation instead: + +``` +MCP(server, mode='auto', prefix='') (plugin: mcp_client) + + Expose the tools from an MCP server as LLM tools. + + Usage: + + MCP("https://example.com/mcp") + MCP("https://example.com/mcp", mode="legacy", prefix="demo_") +``` + +Pass a full toolbox specification to `llm tools` to see the tools provided by a configured instance: + +```bash +llm tools 'MCP("https://datasette.simonwillison.net/-/mcp")' +``` +``` +MCP("https://datasette.simonwillison.net/-/mcp"): + + list_databases(**kwargs) + + List the databases available in this Datasette instance. + + get_database_schema(**kwargs) + + Return the complete SQL schema for a database. + + execute_sql(**kwargs) + + Execute one read-only SQL statement and return its tabular results. +``` + Continuing a conversation with `llm -c` or `llm chat -c` reconstructs any toolboxes from the configuration recorded in the logs, so you don't need to repeat the `-T` option. Each instance is rebuilt fresh, which means any in-memory state a toolbox accumulated during the earlier prompt does not carry over. Toolboxes work well with `llm chat`. Try chatting with the Datasette content database like this: diff --git a/llm/cli.py b/llm/cli.py index f188b7a98..d8163efd5 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -2685,9 +2685,18 @@ def tools(): def tools_list(tool_defs, json_, python_tools): "List available tools that have been provided by plugins" - def introspect_tools(toolbox_class): + def introspect_tools(toolbox): + # Instances report their tools(), which may be generated dynamically. + # Classes can only report tools for their introspectable methods. + if isinstance(toolbox, Toolbox): + if not toolbox._prepared: + toolbox.prepare() + toolbox._prepared = True + tool_iter = toolbox.tools() + else: + tool_iter = toolbox.method_tools() methods = [] - for tool in toolbox_class.method_tools(): + for tool in tool_iter: methods.append( { "name": tool.name, @@ -2698,13 +2707,19 @@ def introspect_tools(toolbox_class): ) return methods + toolbox_specs: dict[int, str] = {} if tool_defs: tools = {} - for tool in _gather_tools(tool_defs, python_tools): + gathered = _gather_tools(tool_defs, python_tools) + # _gather_tools returns --functions tools first, then one per spec + specs = [None] * (len(gathered) - len(tool_defs)) + list(tool_defs) + for spec, tool in zip(specs, gathered): if hasattr(tool, "name"): tools[tool.name] = tool else: tools[tool.__class__.__name__] = tool + if spec is not None and isinstance(tool, Toolbox): + toolbox_specs[id(tool)] = spec else: tools = get_tools() if python_tools: @@ -2715,7 +2730,7 @@ def introspect_tools(toolbox_class): output_tools = [] output_toolboxes = [] tool_objects = [] - toolbox_objects = [] + toolbox_infos = [] for name, tool in sorted(tools.items()): if isinstance(tool, Tool): tool_objects.append(tool) @@ -2728,17 +2743,26 @@ def introspect_tools(toolbox_class): } ) else: - toolbox_objects.append(tool) + toolbox_class = tool if isinstance(tool, type) else tool.__class__ + # Overriding tools() or prepare() means the toolbox generates + # tools at runtime + is_dynamic = any( + getattr(toolbox_class, method) is not getattr(Toolbox, method) + for method in ("tools", "prepare", "prepare_async") + ) + introspected = introspect_tools(tool) + toolbox_infos.append((name, tool, toolbox_class, is_dynamic, introspected)) output_toolboxes.append( { "name": name, + "dynamic": is_dynamic, "tools": [ { - "name": tool["name"], - "description": tool["description"], - "arguments": tool["arguments"], + "name": tool_info["name"], + "description": tool_info["description"], + "arguments": tool_info["arguments"], } - for tool in introspect_tools(tool) + for tool_info in introspected ], } ) @@ -2763,17 +2787,40 @@ def introspect_tools(toolbox_class): ) if tool.description: click.echo(textwrap.indent(tool.description.strip(), " ") + "\n") - for toolbox in toolbox_objects: - click.echo(toolbox.name + ":\n") - for tool in toolbox.method_tools(): - sig = ( - str(inspect.signature(tool.implementation)) - .replace("(self, ", "(") - .replace("(self)", "()") + for name, toolbox, toolbox_class, is_dynamic, introspected in toolbox_infos: + if is_dynamic and isinstance(toolbox, type): + # A dynamic toolbox class has no tools until it is + # instantiated - show its constructor and docstring instead + try: + constructor_sig = str(inspect.signature(toolbox_class)) + except (ValueError, TypeError): + constructor_sig = "(...)" + plugin = getattr(toolbox_class, "plugin", None) + click.echo( + "{}{}{}\n".format( + name, + constructor_sig, + f" (plugin: {plugin})" if plugin else "", + ) ) - click.echo(f" {tool.name}{sig}\n") - if tool.description: - click.echo(textwrap.indent(tool.description.strip(), " ") + "\n") + doc = toolbox_class.__doc__ + if doc: + click.echo(textwrap.indent(inspect.cleandoc(doc), " ") + "\n") + else: + click.echo(toolbox_specs.get(id(toolbox), name) + ":\n") + for tool_info in introspected: + sig = "()" + if tool_info["implementation"]: + sig = ( + str(inspect.signature(tool_info["implementation"])) + .replace("(self, ", "(") + .replace("(self)", "()") + ) + click.echo(f" {tool_info['name']}{sig}\n") + if tool_info["description"]: + click.echo( + textwrap.indent(tool_info["description"].strip(), " ") + "\n" + ) @cli.group( diff --git a/llm/models.py b/llm/models.py index 1c149127b..da6fbc2ef 100644 --- a/llm/models.py +++ b/llm/models.py @@ -2,6 +2,7 @@ import base64 import dataclasses import datetime +import functools import hashlib import re import time @@ -246,6 +247,7 @@ def __init_subclass__(cls, **kwargs): original_init = cls.__init__ + @functools.wraps(original_init) def wrapped_init(self, *args, **kwargs): # Track args/kwargs passed to constructor in self._config # so we can serialize them to a database entry later on diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 1d8fb5f2b..7b8cec56a 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,4 +1,5 @@ import importlib +import inspect import json import pathlib import re @@ -655,6 +656,7 @@ def after_call(tool, tool_call, tool_result): "toolboxes": [ { "name": "Filesystem", + "dynamic": False, "tools": [ { "name": "Filesystem_list_files", @@ -665,6 +667,7 @@ def after_call(tool, tool_call, tool_result): }, { "name": "Memory", + "dynamic": False, "tools": [ { "name": "Memory_append", @@ -899,6 +902,159 @@ def after_call(tool, tool_call, tool_result): plugins.pm.unregister(name="ToolboxPlugin") +class Discovery(llm.Toolbox): + """ + Tools discovered at runtime from a configured source. + + Usage: + + Discovery("demo") + """ + + def __init__(self, source: str, prefix: str = ""): + self.source = source + self.prefix = prefix + + def tools(self): + def greet(name: str) -> str: + "Greet someone by name" + return f"hello {name} from {self.source}" + + yield llm.Tool.function(greet, name=self.prefix + "greet") + + +class Counter(llm.Toolbox): + """ + Registers a counting tool during prepare(). + """ + + def __init__(self, start: int = 0): + self.value = start + + def prepare(self): + def increment() -> int: + "Increment the counter" + self.value += 1 + return self.value + + self.add_tool(increment) + + +class DynamicToolboxPlugin: + __name__ = "DynamicToolboxPlugin" + + @hookimpl + def register_tools(self, register): + register(Discovery) + register(Counter) + + +def test_toolbox_constructor_signature_preserved(): + # Toolbox.__init_subclass__ wraps __init__ - that wrapper should not + # obscure the constructor signature + assert str(inspect.signature(Discovery)) == "(source: str, prefix: str = '')" + assert str(inspect.signature(Filesystem)) == "(path: str)" + + +def test_tools_list_dynamic_toolbox(): + # https://github.com/simonw/llm/issues/1580 + runner = CliRunner() + try: + plugins.pm.register(DynamicToolboxPlugin(), name="DynamicToolboxPlugin") + + # Plain listing shows constructor signature and docstring instead of + # a bare "Discovery:" header with nothing underneath it + result = runner.invoke(cli.cli, ["tools"]) + assert result.exit_code == 0 + assert "Discovery:" not in result.output + assert ( + "Discovery(source: str, prefix: str = '') (plugin: DynamicToolboxPlugin)\n" + "\n" + " Tools discovered at runtime from a configured source.\n" + "\n" + " Usage:\n" + "\n" + ' Discovery("demo")\n' + ) in result.output + + # --json marks the toolbox as dynamic + result2 = runner.invoke(cli.cli, ["tools", "--json"]) + assert result2.exit_code == 0 + toolboxes = { + toolbox["name"]: toolbox + for toolbox in json.loads(result2.output)["toolboxes"] + } + assert toolboxes["Discovery"]["dynamic"] is True + assert toolboxes["Discovery"]["tools"] == [] + + # Passing a constructor spec lists the tools the instance provides, + # with the spec itself as the heading + result3 = runner.invoke(cli.cli, ["tools", 'Discovery("demo")']) + assert result3.exit_code == 0 + assert result3.output == ( + 'Discovery("demo"):\n\n' + " greet(name: str) -> str\n\n" + " Greet someone by name\n\n" + ) + + # And --json with the spec includes the discovered tools + result4 = runner.invoke(cli.cli, ["tools", 'Discovery("demo")', "--json"]) + assert result4.exit_code == 0 + assert json.loads(result4.output)["toolboxes"] == [ + { + "name": "Discovery", + "dynamic": True, + "tools": [ + { + "name": "greet", + "description": "Greet someone by name", + "arguments": { + "properties": {"name": {"type": "string"}}, + "required": ["name"], + "type": "object", + }, + } + ], + } + ] + finally: + plugins.pm.unregister(name="DynamicToolboxPlugin") + + +def test_tools_list_prepare_toolbox(): + # Toolboxes that discover their tools in prepare() are dynamic too + runner = CliRunner() + try: + plugins.pm.register(DynamicToolboxPlugin(), name="DynamicToolboxPlugin") + + result = runner.invoke(cli.cli, ["tools"]) + assert result.exit_code == 0 + assert "Counter:" not in result.output + assert ( + "Counter(start: int = 0) (plugin: DynamicToolboxPlugin)\n" + "\n" + " Registers a counting tool during prepare().\n" + ) in result.output + + result2 = runner.invoke(cli.cli, ["tools", "--json"]) + assert result2.exit_code == 0 + toolboxes = { + toolbox["name"]: toolbox + for toolbox in json.loads(result2.output)["toolboxes"] + } + assert toolboxes["Counter"]["dynamic"] is True + assert toolboxes["Counter"]["tools"] == [] + + # A constructor spec runs prepare() and lists the registered tools + result3 = runner.invoke(cli.cli, ["tools", "Counter(5)"]) + assert result3.exit_code == 0 + assert result3.output == ( + "Counter(5):\n\n increment() -> int\n\n Increment the counter\n\n" + ) + finally: + plugins.pm.unregister(name="DynamicToolboxPlugin") + + def test_register_toolbox_fails_on_bad_class(): class BadTools: def bad(self): From 1b99533ff83aaed70a74dd70781dd4329fb350bb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 2 Aug 2026 16:14:06 -0700 Subject: [PATCH 204/258] llm tools now displays dynamic toolboxes usefully, closes #1580 --- docs/changelog.md | 5 ++ docs/python-api.md | 2 + docs/usage.md | 34 +++++++++ llm/cli.py | 85 ++++++++++++++++++----- llm/models.py | 2 + tests/test_plugins.py | 156 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 265 insertions(+), 19 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 4b31853ba..b5ff986f5 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,10 @@ # Changelog +(unreleased)= +## Unreleased + +- `llm tools` now provides improved output for dynamic toolboxes - toolbox classes such as `MCP` from [llm-mcp-client](https://github.com/simonw/llm-mcp-client) that generate their tools at runtime. These were previously listed as just their name. They are now listed with their constructor signature and class docstring. Passing one or more specifications such as `llm tools 'MCP("https://datasette.simonwillison.net/-/mcp")'` instantiates each toolbox and lists the tools that configured instance provides. `llm tools --json` output now includes a `"dynamic"` boolean key for each toolbox. [#1580](https://github.com/simonw/llm/issues/1580) + (v0_32_rc2)= ## 0.32rc2 (2026-07-30) diff --git a/docs/python-api.md b/docs/python-api.md index 26a237598..88a614a4e 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -401,6 +401,8 @@ In asynchronous contexts the alternative method `await toolbox.prepare_async()` If you want to prepare the class in this way such that it can be used in both synchronous and asynchronous contexts, implement both `prepare()` and `prepare_async()` methods. +Toolbox classes that override `tools()`, `prepare()` or `prepare_async()` are treated as dynamic by the `llm tools` command. Since their full list of tools cannot be determined until an instance has been configured, `llm tools` displays their constructor signature and class docstring instead of a list of tools - so give your dynamic toolbox a descriptive docstring that shows how to configure it. Users can then pass a full specification such as `llm tools 'MCP("https://example.com/mcp")'` to instantiate the toolbox - running `prepare()` if it is implemented - and list the tools that instance provides. See {ref}`the tools usage documentation ` for details. + (python-api-schemas)= ### Schemas diff --git a/docs/usage.md b/docs/usage.md index 00461354f..72af52963 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -207,6 +207,40 @@ Toolboxes always start with a capital letter. They can be configured by passing - Single JSON value: `ToolboxName("hello")` or `ToolboxName([1,2,3])` - Key-value pairs: `ToolboxName(name="test", count=5, items=[1,2])` - treated the same as `{"name": "test", "count": 5, "items": [1, 2]}`, all values must be valid JSON +The `llm tools` command lists toolboxes along with the tools they provide. Some toolboxes cannot know their list of tools until they have been configured - [llm-mcp-client](https://github.com/simonw/llm-mcp-client) for example fetches its tools from whichever MCP server it is pointed at. These dynamic toolboxes are listed with their constructor signature and documentation instead: + +``` +MCP(server, mode='auto', prefix='') (plugin: mcp_client) + + Expose the tools from an MCP server as LLM tools. + + Usage: + + MCP("https://example.com/mcp") + MCP("https://example.com/mcp", mode="legacy", prefix="demo_") +``` + +Pass a full toolbox specification to `llm tools` to see the tools provided by a configured instance: + +```bash +llm tools 'MCP("https://datasette.simonwillison.net/-/mcp")' +``` +``` +MCP("https://datasette.simonwillison.net/-/mcp"): + + list_databases(**kwargs) + + List the databases available in this Datasette instance. + + get_database_schema(**kwargs) + + Return the complete SQL schema for a database. + + execute_sql(**kwargs) + + Execute one read-only SQL statement and return its tabular results. +``` + Continuing a conversation with `llm -c` or `llm chat -c` reconstructs any toolboxes from the configuration recorded in the logs, so you don't need to repeat the `-T` option. Each instance is rebuilt fresh, which means any in-memory state a toolbox accumulated during the earlier prompt does not carry over. Toolboxes work well with `llm chat`. Try chatting with the Datasette content database like this: diff --git a/llm/cli.py b/llm/cli.py index f188b7a98..d8163efd5 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -2685,9 +2685,18 @@ def tools(): def tools_list(tool_defs, json_, python_tools): "List available tools that have been provided by plugins" - def introspect_tools(toolbox_class): + def introspect_tools(toolbox): + # Instances report their tools(), which may be generated dynamically. + # Classes can only report tools for their introspectable methods. + if isinstance(toolbox, Toolbox): + if not toolbox._prepared: + toolbox.prepare() + toolbox._prepared = True + tool_iter = toolbox.tools() + else: + tool_iter = toolbox.method_tools() methods = [] - for tool in toolbox_class.method_tools(): + for tool in tool_iter: methods.append( { "name": tool.name, @@ -2698,13 +2707,19 @@ def introspect_tools(toolbox_class): ) return methods + toolbox_specs: dict[int, str] = {} if tool_defs: tools = {} - for tool in _gather_tools(tool_defs, python_tools): + gathered = _gather_tools(tool_defs, python_tools) + # _gather_tools returns --functions tools first, then one per spec + specs = [None] * (len(gathered) - len(tool_defs)) + list(tool_defs) + for spec, tool in zip(specs, gathered): if hasattr(tool, "name"): tools[tool.name] = tool else: tools[tool.__class__.__name__] = tool + if spec is not None and isinstance(tool, Toolbox): + toolbox_specs[id(tool)] = spec else: tools = get_tools() if python_tools: @@ -2715,7 +2730,7 @@ def introspect_tools(toolbox_class): output_tools = [] output_toolboxes = [] tool_objects = [] - toolbox_objects = [] + toolbox_infos = [] for name, tool in sorted(tools.items()): if isinstance(tool, Tool): tool_objects.append(tool) @@ -2728,17 +2743,26 @@ def introspect_tools(toolbox_class): } ) else: - toolbox_objects.append(tool) + toolbox_class = tool if isinstance(tool, type) else tool.__class__ + # Overriding tools() or prepare() means the toolbox generates + # tools at runtime + is_dynamic = any( + getattr(toolbox_class, method) is not getattr(Toolbox, method) + for method in ("tools", "prepare", "prepare_async") + ) + introspected = introspect_tools(tool) + toolbox_infos.append((name, tool, toolbox_class, is_dynamic, introspected)) output_toolboxes.append( { "name": name, + "dynamic": is_dynamic, "tools": [ { - "name": tool["name"], - "description": tool["description"], - "arguments": tool["arguments"], + "name": tool_info["name"], + "description": tool_info["description"], + "arguments": tool_info["arguments"], } - for tool in introspect_tools(tool) + for tool_info in introspected ], } ) @@ -2763,17 +2787,40 @@ def introspect_tools(toolbox_class): ) if tool.description: click.echo(textwrap.indent(tool.description.strip(), " ") + "\n") - for toolbox in toolbox_objects: - click.echo(toolbox.name + ":\n") - for tool in toolbox.method_tools(): - sig = ( - str(inspect.signature(tool.implementation)) - .replace("(self, ", "(") - .replace("(self)", "()") + for name, toolbox, toolbox_class, is_dynamic, introspected in toolbox_infos: + if is_dynamic and isinstance(toolbox, type): + # A dynamic toolbox class has no tools until it is + # instantiated - show its constructor and docstring instead + try: + constructor_sig = str(inspect.signature(toolbox_class)) + except (ValueError, TypeError): + constructor_sig = "(...)" + plugin = getattr(toolbox_class, "plugin", None) + click.echo( + "{}{}{}\n".format( + name, + constructor_sig, + f" (plugin: {plugin})" if plugin else "", + ) ) - click.echo(f" {tool.name}{sig}\n") - if tool.description: - click.echo(textwrap.indent(tool.description.strip(), " ") + "\n") + doc = toolbox_class.__doc__ + if doc: + click.echo(textwrap.indent(inspect.cleandoc(doc), " ") + "\n") + else: + click.echo(toolbox_specs.get(id(toolbox), name) + ":\n") + for tool_info in introspected: + sig = "()" + if tool_info["implementation"]: + sig = ( + str(inspect.signature(tool_info["implementation"])) + .replace("(self, ", "(") + .replace("(self)", "()") + ) + click.echo(f" {tool_info['name']}{sig}\n") + if tool_info["description"]: + click.echo( + textwrap.indent(tool_info["description"].strip(), " ") + "\n" + ) @cli.group( diff --git a/llm/models.py b/llm/models.py index 1c149127b..da6fbc2ef 100644 --- a/llm/models.py +++ b/llm/models.py @@ -2,6 +2,7 @@ import base64 import dataclasses import datetime +import functools import hashlib import re import time @@ -246,6 +247,7 @@ def __init_subclass__(cls, **kwargs): original_init = cls.__init__ + @functools.wraps(original_init) def wrapped_init(self, *args, **kwargs): # Track args/kwargs passed to constructor in self._config # so we can serialize them to a database entry later on diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 1d8fb5f2b..7b8cec56a 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,4 +1,5 @@ import importlib +import inspect import json import pathlib import re @@ -655,6 +656,7 @@ def after_call(tool, tool_call, tool_result): "toolboxes": [ { "name": "Filesystem", + "dynamic": False, "tools": [ { "name": "Filesystem_list_files", @@ -665,6 +667,7 @@ def after_call(tool, tool_call, tool_result): }, { "name": "Memory", + "dynamic": False, "tools": [ { "name": "Memory_append", @@ -899,6 +902,159 @@ def after_call(tool, tool_call, tool_result): plugins.pm.unregister(name="ToolboxPlugin") +class Discovery(llm.Toolbox): + """ + Tools discovered at runtime from a configured source. + + Usage: + + Discovery("demo") + """ + + def __init__(self, source: str, prefix: str = ""): + self.source = source + self.prefix = prefix + + def tools(self): + def greet(name: str) -> str: + "Greet someone by name" + return f"hello {name} from {self.source}" + + yield llm.Tool.function(greet, name=self.prefix + "greet") + + +class Counter(llm.Toolbox): + """ + Registers a counting tool during prepare(). + """ + + def __init__(self, start: int = 0): + self.value = start + + def prepare(self): + def increment() -> int: + "Increment the counter" + self.value += 1 + return self.value + + self.add_tool(increment) + + +class DynamicToolboxPlugin: + __name__ = "DynamicToolboxPlugin" + + @hookimpl + def register_tools(self, register): + register(Discovery) + register(Counter) + + +def test_toolbox_constructor_signature_preserved(): + # Toolbox.__init_subclass__ wraps __init__ - that wrapper should not + # obscure the constructor signature + assert str(inspect.signature(Discovery)) == "(source: str, prefix: str = '')" + assert str(inspect.signature(Filesystem)) == "(path: str)" + + +def test_tools_list_dynamic_toolbox(): + # https://github.com/simonw/llm/issues/1580 + runner = CliRunner() + try: + plugins.pm.register(DynamicToolboxPlugin(), name="DynamicToolboxPlugin") + + # Plain listing shows constructor signature and docstring instead of + # a bare "Discovery:" header with nothing underneath it + result = runner.invoke(cli.cli, ["tools"]) + assert result.exit_code == 0 + assert "Discovery:" not in result.output + assert ( + "Discovery(source: str, prefix: str = '') (plugin: DynamicToolboxPlugin)\n" + "\n" + " Tools discovered at runtime from a configured source.\n" + "\n" + " Usage:\n" + "\n" + ' Discovery("demo")\n' + ) in result.output + + # --json marks the toolbox as dynamic + result2 = runner.invoke(cli.cli, ["tools", "--json"]) + assert result2.exit_code == 0 + toolboxes = { + toolbox["name"]: toolbox + for toolbox in json.loads(result2.output)["toolboxes"] + } + assert toolboxes["Discovery"]["dynamic"] is True + assert toolboxes["Discovery"]["tools"] == [] + + # Passing a constructor spec lists the tools the instance provides, + # with the spec itself as the heading + result3 = runner.invoke(cli.cli, ["tools", 'Discovery("demo")']) + assert result3.exit_code == 0 + assert result3.output == ( + 'Discovery("demo"):\n\n' + " greet(name: str) -> str\n\n" + " Greet someone by name\n\n" + ) + + # And --json with the spec includes the discovered tools + result4 = runner.invoke(cli.cli, ["tools", 'Discovery("demo")', "--json"]) + assert result4.exit_code == 0 + assert json.loads(result4.output)["toolboxes"] == [ + { + "name": "Discovery", + "dynamic": True, + "tools": [ + { + "name": "greet", + "description": "Greet someone by name", + "arguments": { + "properties": {"name": {"type": "string"}}, + "required": ["name"], + "type": "object", + }, + } + ], + } + ] + finally: + plugins.pm.unregister(name="DynamicToolboxPlugin") + + +def test_tools_list_prepare_toolbox(): + # Toolboxes that discover their tools in prepare() are dynamic too + runner = CliRunner() + try: + plugins.pm.register(DynamicToolboxPlugin(), name="DynamicToolboxPlugin") + + result = runner.invoke(cli.cli, ["tools"]) + assert result.exit_code == 0 + assert "Counter:" not in result.output + assert ( + "Counter(start: int = 0) (plugin: DynamicToolboxPlugin)\n" + "\n" + " Registers a counting tool during prepare().\n" + ) in result.output + + result2 = runner.invoke(cli.cli, ["tools", "--json"]) + assert result2.exit_code == 0 + toolboxes = { + toolbox["name"]: toolbox + for toolbox in json.loads(result2.output)["toolboxes"] + } + assert toolboxes["Counter"]["dynamic"] is True + assert toolboxes["Counter"]["tools"] == [] + + # A constructor spec runs prepare() and lists the registered tools + result3 = runner.invoke(cli.cli, ["tools", "Counter(5)"]) + assert result3.exit_code == 0 + assert result3.output == ( + "Counter(5):\n\n increment() -> int\n\n Increment the counter\n\n" + ) + finally: + plugins.pm.unregister(name="DynamicToolboxPlugin") + + def test_register_toolbox_fails_on_bad_class(): class BadTools: def bad(self): From f043a3b4bc96c250bee3c0ef5920dec58bbf28fd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 2 Aug 2026 16:44:54 -0700 Subject: [PATCH 205/258] Condense echoed tool definitions in response_json too Providers echo the full tool definitions back in every response payload. On a real tool-using turn that echo was 40% of the stored payload, led by long tool descriptions - strings the tools table already holds, linked to the turn via turn_tools. _payload_replacements now takes the turn's tools as (name, description) pairs - response.prompt.tools on the way in, the turn_tools join on the way out - and offers descriptions of 64+ characters as {"$": "tool.NAME.description"} references. A name carrying two different long descriptions within one turn is dropped by an order-independent rule, so both sides reach the same verdict from their own view of the pairs. Parameter schemas are deliberately not offered: providers echo a transformed schema (OpenAI strict mode adds additionalProperties and strict keys), so the stored form would not match byte-for-byte. Measured live on a gpt-5-mini tool call: 4,784 B raw -> 1,856 B stored. Co-Authored-By: Claude Fable 5 --- docs/logging.md | 2 +- llm/logs.py | 89 +++++++++++++++++++++++++++++++--------- tests/test_logs_store.py | 66 +++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+), 20 deletions(-) diff --git a/docs/logging.md b/docs/logging.md index 907393655..bb84e7846 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -536,7 +536,7 @@ Attachments work the same way: the binary content lives in the `attachments` tab The parts of a response are a normalized view of what the provider returned - enough to replay the conversation, but not the whole story. The raw `response.json()` dictionary is also recorded on the turn, in the `turns.response_json` column, so the details that have no part equivalent - usage breakdowns, system fingerprints, logprobs, settings echoes - survive too. -It is stored *condensed*, using [condense-json](https://github.com/simonw/condense-json). Any string of 64 characters or more that also appears in the turn's own messages - the response text, a reasoning summary and its encrypted counterpart, long tool arguments - is replaced with a reference such as `{"$": "0.0.text"}`, keyed by the message offset, part position and field the string came from. The replacement mapping is never stored: it is rebuilt from the stored messages whenever the payload is read, which is sound because message content is frozen by its hash. What remains in the column is roughly the provider envelope, at a fraction of the size of the raw payload and without a second copy of anything the parts already hold. +It is stored *condensed*, using [condense-json](https://github.com/simonw/condense-json). Any string of 64 characters or more that also appears in the turn's own messages - the response text, a reasoning summary and its encrypted counterpart, long tool arguments - is replaced with a reference such as `{"$": "0.0.text"}`, keyed by the message offset, part position and field the string came from. Tool definitions get the same treatment: providers echo every tool back in the payload on every call, so a long tool description is stored once in the `tools` table and referenced as `{"$": "tool.NAME.description"}`. The replacement mapping is never stored: it is rebuilt whenever the payload is read, from the stored messages (frozen by their hashes) and the turn's `turn_tools` rows. What remains in the column is roughly the provider envelope, at a fraction of the size of the raw payload and without a second copy of anything the store already holds. `llm logs --json` resolves the stored payload back to exactly what the provider sent and includes it as `response_json`. From Python, `LogStore.turn_response_json(turn_id)` returns the same resolved dictionary. Turns logged before this column existed, and models that expose no raw payload, record NULL. diff --git a/llm/logs.py b/llm/logs.py index f5ad64309..e5f7e9a88 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -516,7 +516,9 @@ def _log_in_transaction(self, response, thread_id: str | None) -> str: "duration_ms": response.duration_ms(), "datetime_utc": response.datetime_utc(), "response_json": condense_payload( - getattr(response, "response_json", None), own_messages + getattr(response, "response_json", None), + own_messages, + [(tool.name, tool.description) for tool in response.prompt.tools], ), }, replace=True, @@ -624,7 +626,15 @@ def turn_response_json(self, turn_id: str) -> Any: return None inputs = self.load_chain(row["parent_message_hash"]) outputs = self.load_chain(row["tip_message_hash"])[len(inputs) :] - return resolve_payload(row["response_json"], outputs) + return resolve_payload( + row["response_json"], outputs, self._turn_tool_pairs(turn_id) + ) + + def _turn_tool_pairs(self, turn_id: str) -> list[tuple[str, str]]: + return [ + (row["name"], row["description"]) + for row in self.db.query(TURN_TOOLS_SQL, [turn_id]) + ] # -- verification -------------------------------------------------- @@ -821,29 +831,47 @@ def _now() -> str: # -- condensed provider payloads ---------------------------------------- # -# The raw response.json() payload mostly duplicates content the message -# tables already hold: the response text, reasoning summaries and their -# encrypted blobs, long tool arguments. The turn stores it condensed -# instead - strings that already live in the turn's own messages are -# swapped for {"$": key} references (condense-json), leaving roughly the -# provider envelope: ids, usage, fingerprints, settings echoes. The -# replacement dict is never stored; it is rebuilt from the chain segment -# on the way out, which is sound because message content is hash-frozen, -# so the same walk over the same messages produces the same dict on both -# sides of the round trip. +# The raw response.json() payload mostly duplicates content the store +# already holds: the response text, reasoning summaries and their +# encrypted blobs, long tool arguments, and the tool definitions the +# provider echoes back on every call. The turn stores it condensed +# instead - strings that already live in the turn's own messages or its +# tools are swapped for {"$": key} references (condense-json), leaving +# roughly the provider envelope: ids, usage, fingerprints, settings +# echoes. The replacement dict is never stored; it is rebuilt on the way +# out from the chain segment (hash-frozen) and the turn_tools join +# (per-turn provenance), so the same walk over the same rows produces +# the same dict on both sides of the round trip. # Below this a {"$": key} marker costs about as much as the string it # replaces. _CONDENSE_MIN_LENGTH = 64 +# The tools a turn was given, as the (name, description) pairs +# _payload_replacements expects. +TURN_TOOLS_SQL = """ +select tools.name, tools.description +from turn_tools join tools on tools.id = turn_tools.tool_id +where turn_tools.turn_id = ? +""" + -def _payload_replacements(messages) -> dict[str, str]: +def _payload_replacements(messages, tools=()) -> dict[str, str]: """Replacement strings for condensing a turn's provider payload. ``messages`` is the turn's own contribution - the chain segment between its parent and tip. Keys are structural (message offset, part position, field path) so the identical dict can be rebuilt from the stored segment at read time. + + ``tools`` is the turn's tools as (name, description) pairs - + ``response.prompt.tools`` on the way in, the ``turn_tools`` join on + the way out. Descriptions are keyed by tool name; a name carrying + two different descriptions in one turn is dropped, the same + order-independent verdict from either side's view of the pairs. + (Parameter schemas are deliberately not offered: providers echo a + transformed schema - OpenAI strict mode adds keys - so the stored + form would not match.) """ replacements: dict[str, str] = {} @@ -879,17 +907,34 @@ def walk(prefix: str, obj: Any) -> None: add(f"{base}.args2", spaced) walk(f"{base}.pm", part.get("provider_metadata") or {}) walk(f"{mi}.pm", message_dict.get("provider_metadata") or {}) + + descriptions: dict[str, str] = {} + conflicting = set() + for name, description in tools: + if ( + not name + or not isinstance(description, str) + or len(description) < _CONDENSE_MIN_LENGTH + ): + continue + if descriptions.get(name, description) != description: + conflicting.add(name) + continue + descriptions[name] = description + for name, description in descriptions.items(): + if name not in conflicting: + replacements[f"tool.{name}.description"] = description return replacements -def condense_payload(payload: Any, messages) -> str | None: - "JSON text of ``payload`` with strings from ``messages`` condensed." +def condense_payload(payload: Any, messages, tools=()) -> str | None: + "JSON text of ``payload`` with strings from ``messages`` and ``tools`` condensed." if payload is None: return None - return json.dumps(condense_json(payload, _payload_replacements(messages))) + return json.dumps(condense_json(payload, _payload_replacements(messages, tools))) -def resolve_payload(condensed: str | None, messages) -> Any: +def resolve_payload(condensed: str | None, messages, tools=()) -> Any: """Reverse of :func:`condense_payload`. Raises ``condense_json.UncondenseError`` when the stored payload @@ -897,7 +942,9 @@ def resolve_payload(condensed: str | None, messages) -> Any: """ if condensed is None: return None - return uncondense_json(json.loads(condensed), _payload_replacements(messages)) + return uncondense_json( + json.loads(condensed), _payload_replacements(messages, tools) + ) # -- llm logs support --------------------------------------------------- @@ -1101,7 +1148,11 @@ def _resolve_response_json(self, row: dict, outputs) -> str | None: if not condensed: return None try: - return json.dumps(resolve_payload(condensed, outputs)) + return json.dumps( + resolve_payload( + condensed, outputs, self.store._turn_tool_pairs(row["id"]) + ) + ) except UncondenseError: return None diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index b9ca29c23..c84f5f92d 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -1848,3 +1848,69 @@ def test_short_strings_are_excluded(self): from llm.logs import _payload_replacements assert _payload_replacements([llm.assistant("short")]) == {} + + def test_tool_descriptions_are_offered(self): + from llm.logs import _payload_replacements + + description = "d" * 64 + replacements = _payload_replacements([], tools=[("search", description)]) + assert replacements == {"tool.search.description": description} + + def test_conflicting_tool_descriptions_are_dropped(self): + from llm.logs import _payload_replacements + + # Whichever order the pairs arrive in, a name that carries two + # different long descriptions is excluded on both sides. + pairs = [("search", "a" * 64), ("search", "b" * 64), ("other", "c" * 64)] + for ordering in (pairs, list(reversed(pairs))): + assert _payload_replacements([], tools=ordering) == { + "tool.other.description": "c" * 64 + } + + def test_short_or_missing_tool_descriptions_are_excluded(self): + from llm.logs import _payload_replacements + + assert _payload_replacements([], tools=[("a", "short"), ("b", None)]) == {} + + +class TestResponseJsonToolEcho: + """Providers echo the turn's tool definitions back in the payload; + long descriptions condense against the tools table.""" + + DESCRIPTION = ( + "Execute JavaScript code using a persistent context. State is " + "maintained between calls, allowing variables to persist." + ) + + def logged(self, store, mock_model): + def execute_javascript(javascript: str) -> str: + return "ran" + + tool = llm.Tool.function(execute_javascript, description=self.DESCRIPTION) + mock_model.enqueue(["ok"]) + response = mock_model.prompt("run it", tools=[tool]) + response.text() + response.response_json = { + "content": "ok", + "tools": [ + { + "name": "execute_javascript", + "description": self.DESCRIPTION, + "parameters": {"type": "object"}, + } + ], + } + return store.log(response), response.response_json + + def test_the_echoed_description_is_a_reference(self, store, mock_model): + turn_id, _ = self.logged(store, mock_model) + stored = json.loads(store.db["turns"].get(turn_id)["response_json"]) + assert stored["tools"][0]["description"] == { + "$": "tool.execute_javascript.description" + } + + def test_the_payload_resolves_via_the_turn_tools_join(self, store, mock_model): + turn_id, payload = self.logged(store, mock_model) + assert store.turn_response_json(turn_id) == payload + (row,) = merged_log_rows(store) + assert json.loads(row["response_json"]) == payload From 71ba7f1e67ed178a2eaa29914489115037013a16 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 2 Aug 2026 16:56:18 -0700 Subject: [PATCH 206/258] Changelog: response payloads are persisted again, logprobs included The 0.32 notes said raw payloads were no longer stored anywhere, which stopped being true when turns.response_json landed - logprobs travel in the payload on both the streamed path (combine_chunks collects them) and the non-streamed dump. Co-Authored-By: Claude Fable 5 --- docs/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index b5ff986f5..c1bb79562 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -31,7 +31,7 @@ You can create a backup of your logs database prior to upgrading using: - `llm prompt` logs are now written to a new set of tables, not the legacy tables. - New documented {ref}`Response.log_to_db() ` Python API for writing to a SQLite logs database. - `llm logs` now reads both generations of log tables: history recorded by older versions of LLM in the legacy `responses` table is merged into the output alongside new turns, with rows from the dual-write era deduplicated by id. The `-m`, `-c`, `-f`, `-T`, `--tools` and `--schema` filters work across both. [#1562](https://github.com/simonw/llm/pull/1562) -- Raw provider payloads (the old `prompt_json` and `response_json` columns) are no longer persisted - the stored message chain is the record of what was sent and returned. This means data that only ever lived in those raw payloads is no longer stored, notably the log probabilities returned by OpenAI completion models with `-o logprobs`. +- Raw provider response payloads are recorded on the new tables: the full `response.json()` dictionary is stored in a `turns.response_json` column, {ref}`condensed ` using [condense-json](https://github.com/simonw/condense-json) so that strings the message tables already hold - response text, reasoning summaries and their encrypted blobs, long tool arguments and tool descriptions - are stored as references rather than copies. `llm logs --json` shows the payload resolved back to exactly what the provider sent. This preserves data that only lives in the raw payload, such as the log probabilities returned by OpenAI models with `-o logprobs`. The old `prompt_json` column is no longer persisted - the stored message chain is the record of what was sent. [#1586](https://github.com/simonw/llm/pull/1586) - `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. See {ref}`logging-search`. - `llm logs` now shows which configured toolbox instance provided each tool - in the tools list for a prompt and on each tool result, so a `SQLite_query` result records that it ran against `SQLite("mydb.db")`. - New {ref}`message_tree SQL view ` that renders each conversation thread as an indented text outline, for exploring the message store directly with SQL. [#1562](https://github.com/simonw/llm/pull/1562) From e1267a4e6174f7fedf24aed6d11530b7518e5014 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 2 Aug 2026 22:14:17 -0700 Subject: [PATCH 207/258] Structural payload condensing via condense-json 1.1 condense-json 1.1 matches dict and list replacement values as whole subtrees by structural equality, encodes near-matches of a dict base as the base plus a patch, and lets a payload that mostly consists of known boilerplate collapse to a single reference. The replacement pool for turns.response_json now exploits all of it - dependency bumped to >=1.1. Three new replacement sources: - The turn's JSON schema, when one was used. Providers echo the full schema back in the payload (OpenAI text.format) and the schemas table already stores it once, linked via turns.schema_id - offered structurally, so the echo's key order does not matter. Measured on a live structured-output call: 53% saved became 64%. - Container nodes inside provider_metadata, alongside their leaf strings - outermost match wins, so a payload embedding a whole metadata object (a reasoning summary list) condenses to one reference instead of one per string. - Object-form tool arguments (argsv). OpenAI transports arguments as a JSON-encoded string, which the existing byte-exact serializations cover, but Anthropic and Gemini embed them as objects - structural equality is the only thing that can match those. Verified against a live claude-opus turn. Model classes can also declare a json_replacements dictionary of recurring payload boilerplate - the zstandard-custom-dictionary idea. Entries join the pool under an m. prefix (collision-proof against the derived keys) with no length threshold, since the plugin author curates them, and dict entries double as condense-json merge bases. Resolution looks the model up again through the registry at read time, so a payload condensed against a model's dictionary needs that plugin installed to resolve: an unknown model fails closed - UncondenseError from the API, payload absent from llm logs rows. That lookup is also why dictionaries must be append-only; editing an entry silently breaks every payload already stored against it. Co-Authored-By: Claude Fable 5 --- docs/logging.md | 2 + llm/logs.py | 123 +++++++++++++++++++++++++++----- pyproject.toml | 2 +- tests/test_logs_store.py | 150 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 260 insertions(+), 17 deletions(-) diff --git a/docs/logging.md b/docs/logging.md index bb84e7846..554fab338 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -538,6 +538,8 @@ The parts of a response are a normalized view of what the provider returned - en It is stored *condensed*, using [condense-json](https://github.com/simonw/condense-json). Any string of 64 characters or more that also appears in the turn's own messages - the response text, a reasoning summary and its encrypted counterpart, long tool arguments - is replaced with a reference such as `{"$": "0.0.text"}`, keyed by the message offset, part position and field the string came from. Tool definitions get the same treatment: providers echo every tool back in the payload on every call, so a long tool description is stored once in the `tools` table and referenced as `{"$": "tool.NAME.description"}`. The replacement mapping is never stored: it is rebuilt whenever the payload is read, from the stored messages (frozen by their hashes) and the turn's `turn_tools` rows. What remains in the column is roughly the provider envelope, at a fraction of the size of the raw payload and without a second copy of anything the store already holds. +Model plugins can contribute a *dictionary* of their own recurring boilerplate - the same idea as a zstandard custom dictionary - by defining a `json_replacements` class attribute: a dict mapping keys to payload fragments the plugin author knows appear verbatim in every reply, such as OpenAI's all-zero `tool_usage` accounting block. These are matched like any other replacement (dicts and lists structurally) and resolved by looking the model up again at read time. Two consequences follow: reading such payloads requires the model's plugin to be installed, and entries are **append-only** - never remove or change an existing entry, only add new ones, because editing an entry silently breaks every payload already condensed against it. + `llm logs --json` resolves the stored payload back to exactly what the provider sent and includes it as `response_json`. From Python, `LogStore.turn_response_json(turn_id)` returns the same resolved dictionary. Turns logged before this column existed, and models that expose no raw payload, record NULL. (logging-message-store-tables)= diff --git a/llm/logs.py b/llm/logs.py index e5f7e9a88..744efb9f0 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -519,6 +519,10 @@ def _log_in_transaction(self, response, thread_id: str | None) -> str: getattr(response, "response_json", None), own_messages, [(tool.name, tool.description) for tool in response.prompt.tools], + schema=response.prompt.schema, + model_replacements=getattr( + response.model, "json_replacements", None + ), ), }, replace=True, @@ -615,8 +619,12 @@ def turn_response_json(self, turn_id: str) -> Any: row = next( iter( self.db.query( - "select parent_message_hash, tip_message_hash, response_json" - " from turns where id = ?", + "select turns.model, turns.parent_message_hash," + " turns.tip_message_hash, turns.response_json," + " schemas.content as schema_json" + " from turns" + " left join schemas on turns.schema_id = schemas.id" + " where turns.id = ?", [turn_id], ) ), @@ -627,7 +635,11 @@ def turn_response_json(self, turn_id: str) -> Any: inputs = self.load_chain(row["parent_message_hash"]) outputs = self.load_chain(row["tip_message_hash"])[len(inputs) :] return resolve_payload( - row["response_json"], outputs, self._turn_tool_pairs(turn_id) + row["response_json"], + outputs, + self._turn_tool_pairs(turn_id), + schema=_load(row["schema_json"]), + model_replacements=_model_json_replacements(row["model"]), ) def _turn_tool_pairs(self, turn_id: str) -> list[tuple[str, str]]: @@ -856,8 +868,10 @@ def _now() -> str: """ -def _payload_replacements(messages, tools=()) -> dict[str, str]: - """Replacement strings for condensing a turn's provider payload. +def _payload_replacements( + messages, tools=(), schema=None, model_replacements=None +) -> dict[str, Any]: + """Replacement values for condensing a turn's provider payload. ``messages`` is the turn's own contribution - the chain segment between its parent and tip. Keys are structural (message offset, @@ -871,19 +885,53 @@ def _payload_replacements(messages, tools=()) -> dict[str, str]: order-independent verdict from either side's view of the pairs. (Parameter schemas are deliberately not offered: providers echo a transformed schema - OpenAI strict mode adds keys - so the stored - form would not match.) + form would not match even structurally.) + + ``schema`` is the turn's JSON schema dict, when one was used - the + provider echoes it back in the payload (OpenAI ``text.format``) and + the schemas table already stores it once. It is offered as a + structural value, so the echo's key order does not matter. + + ``model_replacements`` is the model class's ``json_replacements`` + dictionary of common boilerplate - the zstd-custom-dictionary idea: + payload fragments the plugin author knows recur in every reply, + keyed here under an ``m.`` prefix so they can never collide with + the derived keys. These resolve by looking the model up again at + read time, so plugins must treat their ``json_replacements`` as + append-only: removing or changing an entry breaks every payload + already stored against it. + + Values are strings (matched as substrings) and dicts or lists + (matched as whole subtrees by condense-json's structural equality). + Container values inside provider_metadata are offered as well as + their leaf strings - outermost match wins, so a payload that embeds + a whole metadata object (a reasoning ``summary`` list, say) + condenses to one reference instead of one per string. """ - replacements: dict[str, str] = {} + replacements: dict[str, Any] = {} def add(key: str, value: Any) -> None: if isinstance(value, str) and len(value) >= _CONDENSE_MIN_LENGTH: replacements[key] = value + def add_value(key: str, value: Any) -> None: + if isinstance(value, (dict, list)) and value: + # The same length bar as strings, applied to the canonical + # form, so a reference is always a clear win. + try: + size = len(json.dumps(value, separators=(",", ":"))) + except (TypeError, ValueError): + return + if size >= _CONDENSE_MIN_LENGTH: + replacements[key] = value + def walk(prefix: str, obj: Any) -> None: if isinstance(obj, dict): + add_value(prefix, obj) for key, value in obj.items(): walk(f"{prefix}.{key}", value) elif isinstance(obj, list): + add_value(prefix, obj) for index, value in enumerate(obj): walk(f"{prefix}.{index}", value) else: @@ -897,14 +945,17 @@ def walk(prefix: str, obj: Any) -> None: add(f"{base}.output", part.get("output")) arguments = part.get("arguments") if arguments: - # Providers carry tool arguments as a JSON-encoded - # string; OpenAI uses the compact form, so offer both - # serializations of the stored dict. + # Providers that carry tool arguments as a JSON-encoded + # string need a byte-exact serialization - OpenAI uses + # the compact form, so offer both. Providers that embed + # them as an object (Anthropic, Gemini) match the dict + # itself structurally. compact = json.dumps(arguments, separators=(",", ":")) spaced = json.dumps(arguments) add(f"{base}.args", compact) if spaced != compact: add(f"{base}.args2", spaced) + add_value(f"{base}.argsv", arguments) walk(f"{base}.pm", part.get("provider_metadata") or {}) walk(f"{mi}.pm", message_dict.get("provider_metadata") or {}) @@ -924,17 +975,33 @@ def walk(prefix: str, obj: Any) -> None: for name, description in descriptions.items(): if name not in conflicting: replacements[f"tool.{name}.description"] = description + + add_value("schema", schema) + + # Model-declared boilerplate passes through as curated: the plugin + # author chose these entries, so no length threshold applies. + for key, value in (model_replacements or {}).items(): + replacements[f"m.{key}"] = value return replacements -def condense_payload(payload: Any, messages, tools=()) -> str | None: - "JSON text of ``payload`` with strings from ``messages`` and ``tools`` condensed." +def condense_payload( + payload: Any, messages, tools=(), schema=None, model_replacements=None +) -> str | None: + "JSON text of ``payload`` condensed against the turn's stored content." if payload is None: return None - return json.dumps(condense_json(payload, _payload_replacements(messages, tools))) + return json.dumps( + condense_json( + payload, + _payload_replacements(messages, tools, schema, model_replacements), + ) + ) -def resolve_payload(condensed: str | None, messages, tools=()) -> Any: +def resolve_payload( + condensed: str | None, messages, tools=(), schema=None, model_replacements=None +) -> Any: """Reverse of :func:`condense_payload`. Raises ``condense_json.UncondenseError`` when the stored payload @@ -943,10 +1010,30 @@ def resolve_payload(condensed: str | None, messages, tools=()) -> Any: if condensed is None: return None return uncondense_json( - json.loads(condensed), _payload_replacements(messages, tools) + json.loads(condensed), + _payload_replacements(messages, tools, schema, model_replacements), ) +def _model_json_replacements(model_id: str | None): + """The ``json_replacements`` boilerplate dictionary for a model id. + + Resolved through the registry at read time, so payloads condensed + against a model's dictionary need that model's plugin installed to + resolve again. An unknown model returns None; any markers that + depended on it surface as UncondenseError from resolve_payload. + """ + if not model_id: + return None + from llm import UnknownModelError, get_model + + try: + model = get_model(model_id) + except UnknownModelError: + return None + return getattr(model, "json_replacements", None) + + # -- llm logs support --------------------------------------------------- # # Rows shaped like the ones the older `responses` query produced, so the @@ -1150,7 +1237,11 @@ def _resolve_response_json(self, row: dict, outputs) -> str | None: try: return json.dumps( resolve_payload( - condensed, outputs, self.store._turn_tool_pairs(row["id"]) + condensed, + outputs, + self.store._turn_tool_pairs(row["id"]), + schema=_load(row.get("schema_json")), + model_replacements=_model_json_replacements(row.get("model")), ) ) except UncondenseError: diff --git a/pyproject.toml b/pyproject.toml index 37cb32ee1..049a9e61a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ dependencies = [ "click", - "condense-json>=1.0", + "condense-json>=1.1", "openai>=2.32.0", "click-default-group>=1.2.3", "sqlite-utils>=4.0", diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index c84f5f92d..719d7ca7f 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -1818,8 +1818,13 @@ def test_part_strings_are_keyed_by_position(self): ) ] replacements = _payload_replacements(messages) + # Leaf strings by path, plus each provider_metadata container + # offered structurally so a payload embedding the whole object + # condenses to one reference. assert replacements == { "0.0.text": "r" * 70, + "0.0.pm": {"openai": {"encrypted_content": blob}}, + "0.0.pm.openai": {"encrypted_content": blob}, "0.0.pm.openai.encrypted_content": blob, "0.1.text": "t" * 64, } @@ -1914,3 +1919,148 @@ def test_the_payload_resolves_via_the_turn_tools_join(self, store, mock_model): assert store.turn_response_json(turn_id) == payload (row,) = merged_log_rows(store) assert json.loads(row["response_json"]) == payload + + +def _structural_schema(): + return { + "type": "object", + "properties": {"name": {"type": "string", "description": "d" * 40}}, + "required": ["name"], + } + + +class TestResponseJsonStructural: + """Structural replacement values (condense-json 1.1): payload + subtrees that equal stored content condense whole, regardless of + key order or serialization.""" + + def test_schema_echo_condenses_against_the_schemas_table(self, store, mock_model): + schema = _structural_schema() + mock_model.enqueue(['{"name": "Cleo"}']) + response = mock_model.prompt("Extract", schema=schema) + response.text() + # The echo reorders keys, as providers do - structural equality + # still matches + echoed = { + "required": ["name"], + "properties": {"name": {"type": "string", "description": "d" * 40}}, + "type": "object", + } + payload = {"text": {"format": {"type": "json_schema", "schema": echoed}}} + response.response_json = payload + turn_id = store.log(response) + stored = json.loads(store.db["turns"].get(turn_id)["response_json"]) + assert stored["text"]["format"]["schema"] == {"$": "schema"} + # Resolves via the schemas table; the substituted form is the + # stored schema's key order, structurally equal to the echo + resolved = store.turn_response_json(turn_id) + assert resolved["text"]["format"]["schema"] == schema + (row,) = merged_log_rows(store) + row_payload = json.loads(row["response_json"]) + assert row_payload["text"]["format"]["schema"] == schema + + def test_object_form_tool_arguments_condense(self, store, mock_model): + # Anthropic and Gemini embed tool arguments as objects rather + # than JSON-encoded strings + arguments = {"javascript": "x" * 64} + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "run", + tool_results=[ + llm.ToolResult(name="execute", output="42", tool_call_id="t1") + ], + messages=[ + llm.assistant( + llm.parts.ToolCallPart( + name="execute", arguments=arguments, tool_call_id="t1" + ) + ) + ], + ) + response.text() + payload = {"content": [{"type": "tool_use", "input": dict(arguments)}]} + response.response_json = payload + turn_id = store.log(response) + assert store.turn_response_json(turn_id) == payload + + def test_provider_metadata_container_condenses_whole(self, store, mock_model): + summary = [{"type": "summary_text", "text": "s" * 60}] + mock_model.enqueue( + [ + llm.parts.StreamEvent( + type="reasoning", + chunk="thinking", + provider_metadata={"openai": {"summary": summary}}, + ), + "ok", + ] + ) + response = mock_model.prompt("hi") + response.text() + payload = {"output": [{"type": "reasoning", "summary": summary}]} + response.response_json = payload + turn_id = store.log(response) + stored = store.db["turns"].get(turn_id)["response_json"] + # One reference for the whole list, not one per inner string + assert '"$": "0.0.pm.openai.summary"' in stored + assert store.turn_response_json(turn_id) == payload + + +class TestModelJsonReplacements: + """Model classes can declare a json_replacements dictionary of + recurring payload boilerplate - like a zstd custom dictionary - + resolved by looking the model up again at read time.""" + + @staticmethod + def boiler(): + return { + "image_gen": {"input_tokens": 0, "output_tokens": 0}, + "web_search": {"num_requests": 0}, + } + + def test_model_boilerplate_condenses_and_resolves(self, store, mock_model): + mock_model.json_replacements = {"tool_usage_0": self.boiler()} + try: + mock_model.enqueue(["ok"]) + response = mock_model.prompt("hi") + response.text() + payload = { + "content": "ok", + # Key order differs from the declared entry - structural + # matching still applies + "tool_usage": { + "web_search": {"num_requests": 0}, + "image_gen": {"output_tokens": 0, "input_tokens": 0}, + }, + } + response.response_json = payload + turn_id = store.log(response) + stored = store.db["turns"].get(turn_id)["response_json"] + assert json.loads(stored)["tool_usage"] == {"$": "m.tool_usage_0"} + resolved = store.turn_response_json(turn_id) + assert resolved["tool_usage"] == self.boiler() + (row,) = merged_log_rows(store) + assert json.loads(row["response_json"])["tool_usage"] == self.boiler() + finally: + del mock_model.json_replacements + + def test_unknown_model_fails_closed(self, store, mock_model): + mock_model.json_replacements = {"tool_usage_0": self.boiler()} + try: + mock_model.enqueue(["ok"]) + response = mock_model.prompt("hi") + response.text() + response.response_json = {"tool_usage": self.boiler()} + turn_id = store.log(response) + finally: + del mock_model.json_replacements + # The model is gone (or its plugin uninstalled) at read time + store.db.execute( + "update turns set model = 'gone-model' where id = ?", [turn_id] + ) + from condense_json import UncondenseError + + with pytest.raises(UncondenseError): + store.turn_response_json(turn_id) + (row,) = merged_log_rows(store) + assert row["response_json"] is None From 22b9b44b97ae579608ce743612667b748b3855f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 2 Aug 2026 22:14:52 -0700 Subject: [PATCH 208/258] OpenAI boilerplate dictionaries for payload condensing json_replacements for both OpenAI API shapes, mined from real payloads rather than guessed: chat completions get the all-zero usage-details blocks, the Responses API gets the tool_usage accounting block, cached-token details, the reasoning settings echo in its two observed variants, the default text block, and response_env_0 - the fourteen static top-level envelope keys, which dict-entry merge basing turns into the big win: a payload whose top level mostly matches stores as one reference plus its varying keys. Settings a config changes (top_p, prompt_cache_retention) simply ride in the patch. Measured on real logged turns: a minimal "Hi" reply that previously condensed 0% (nothing in it existed elsewhere in the store) now stores at 814 bytes from 1,372 raw, and a tool-call turn at 1,362 from 3,463 - 51% across the sample, up from 39% before the dictionaries and 22% with string matching alone. Both dictionaries carry the warning that matters: NEVER remove or change an existing entry, only append - stored payloads resolve against these by key, so an edit silently breaks every payload already condensed against it. Variant blocks get appended with new suffixes (reasoning_settings_2 and so on) rather than edited. Co-Authored-By: Claude Fable 5 --- docs/changelog.md | 2 +- llm/default_plugins/openai_models.py | 97 +++++++++++++++++++++++++++- 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index c1bb79562..03463cd8e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -31,7 +31,7 @@ You can create a backup of your logs database prior to upgrading using: - `llm prompt` logs are now written to a new set of tables, not the legacy tables. - New documented {ref}`Response.log_to_db() ` Python API for writing to a SQLite logs database. - `llm logs` now reads both generations of log tables: history recorded by older versions of LLM in the legacy `responses` table is merged into the output alongside new turns, with rows from the dual-write era deduplicated by id. The `-m`, `-c`, `-f`, `-T`, `--tools` and `--schema` filters work across both. [#1562](https://github.com/simonw/llm/pull/1562) -- Raw provider response payloads are recorded on the new tables: the full `response.json()` dictionary is stored in a `turns.response_json` column, {ref}`condensed ` using [condense-json](https://github.com/simonw/condense-json) so that strings the message tables already hold - response text, reasoning summaries and their encrypted blobs, long tool arguments and tool descriptions - are stored as references rather than copies. `llm logs --json` shows the payload resolved back to exactly what the provider sent. This preserves data that only lives in the raw payload, such as the log probabilities returned by OpenAI models with `-o logprobs`. The old `prompt_json` column is no longer persisted - the stored message chain is the record of what was sent. [#1586](https://github.com/simonw/llm/pull/1586) +- Raw provider response payloads are recorded on the new tables: the full `response.json()` dictionary is stored in a `turns.response_json` column, {ref}`condensed ` using [condense-json](https://github.com/simonw/condense-json) 1.1 so that content the database already holds is stored as references rather than copies - response text, reasoning summaries and their encrypted blobs, tool arguments and descriptions, and the JSON schema echoed back on structured-output calls, matched structurally so provider key ordering never matters. Model plugins can declare a `json_replacements` dictionary of recurring payload boilerplate, and a payload that mostly consists of a known envelope stores as that base plus a patch. `llm logs --json` shows the payload resolved back to exactly what the provider sent. This preserves data that only lives in the raw payload, such as the log probabilities returned by OpenAI models with `-o logprobs`. The old `prompt_json` column is no longer persisted - the stored message chain is the record of what was sent. [#1586](https://github.com/simonw/llm/pull/1586) - `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. See {ref}`logging-search`. - `llm logs` now shows which configured toolbox instance provided each tool - in the tools list for a prompt and on each tool result, so a `SQLite_query` result records that it ran against `SQLite("mydb.db")`. - New {ref}`message_tree SQL view ` that renders each conversation thread as an indented text outline, for exploring the message store directly with SQL. [#1562](https://github.com/simonw/llm/pull/1562) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 1a5770ba2..e444e2edf 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -4,7 +4,7 @@ import sys from collections.abc import AsyncGenerator, Iterable, Iterator from enum import Enum -from typing import Any +from typing import Any, ClassVar import click import httpx @@ -1008,6 +1008,29 @@ def _attachment(attachment, image_detail=None): class _Shared: + # Boilerplate fragments that recur verbatim in this API's response + # payloads, offered to the log store's payload condensing as shared + # replacements - the same idea as a zstandard custom dictionary. + # Dict and list values match payload subtrees structurally, so key + # order does not matter. + # + # NEVER remove or change an existing entry - only ever append new + # ones. Stored payloads reference these by key and resolve against + # the dictionary at read time, so editing an entry silently breaks + # every payload already condensed against it. + json_replacements: ClassVar[dict] = { + "completion_tokens_details_0": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details_0": { + "audio_tokens": 0, + "cached_tokens": 0, + }, + } + def __init__( self, model_id, @@ -1495,6 +1518,78 @@ def _responses_attachment(attachment, image_detail=None): class _SharedResponses(_Shared): """Mixin that translates llm.Prompt into Responses API parameters.""" + # Recurring boilerplate in Responses API payloads. Same contract as + # _Shared.json_replacements, which this replaces for Responses + # models: NEVER remove or change an existing entry - only ever + # append new ones, or payloads already stored against these keys + # stop resolving. + json_replacements: ClassVar[dict] = { + "tool_usage_0": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0, + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0, + }, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "input_tokens_details_0": { + "cached_tokens": 0, + "cache_write_tokens": 0, + }, + # The reasoning settings echo, in the variants the API emits + # depending on reasoning context. New variants get appended as + # reasoning_settings_2 and so on - never edited in place. + "reasoning_settings_0": { + "effort": "medium", + "summary": "detailed", + "context": "all_turns", + "mode": "standard", + }, + "reasoning_settings_1": { + "effort": "medium", + "summary": "detailed", + "context": "current_turn", + "mode": "standard", + }, + # The default text block on non-schema replies + "text_format_0": {"format": {"type": "text"}, "verbosity": "medium"}, + # The static envelope of a Responses payload. Dict entries also + # act as condense-json merge bases: a payload top-level that + # differs from this in a few keys stores as this base plus a + # patch, so settings a config changes (top_p, say) simply ride + # in the patch. + "response_env_0": { + "object": "response", + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "top_p": 1.0, + "background": False, + "service_tier": "default", + "status": "completed", + "top_logprobs": 0, + "truncation": "disabled", + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "store": False, + "tools": [], + }, + "message_completed": { + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "final_answer", + }, + } + def __str__(self) -> str: return f"OpenAI Responses: {self.model_id}" From 31df473e928e842d27c275cf7350665e33ca2e59 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 2 Aug 2026 22:30:12 -0700 Subject: [PATCH 209/258] Changelog: response_json work moves to the Unreleased section The condensed-payload bullet had been edited in place inside the 0.32rc1 notes, but rc1 shipped without payload persistence - its original "no longer persisted" text was true for that release and is restored. The Unreleased section now carries three entries for PR #1586: the restored turns.response_json column with condense-json 1.1 structural matching, the json_replacements model dictionaries with measured savings, and the streamed-reasoning ciphertext fix. Co-Authored-By: Claude Fable 5 --- docs/changelog.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 03463cd8e..dde8845bb 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,9 @@ (unreleased)= ## Unreleased +- Raw provider response payloads are persisted again: the full `response.json()` dictionary is stored in a new `turns.response_json` column, {ref}`condensed ` using [condense-json](https://github.com/simonw/condense-json) 1.1 so that content the database already holds is stored as references rather than copies - response text, reasoning summaries and their encrypted blobs, tool arguments and descriptions, and the JSON schema echoed back on structured-output calls, matched structurally so provider key ordering never matters. `llm logs --json` shows the payload resolved back to exactly what the provider sent, and `LogStore.turn_response_json(turn_id)` returns it from Python. This restores data that only lives in the raw payload, such as the log probabilities returned by OpenAI models with `-o logprobs`. `prompt_json` remains unpersisted - the stored message chain is the record of what was sent. [#1586](https://github.com/simonw/llm/pull/1586) +- Model plugins can declare a `json_replacements` class attribute: a dictionary of payload fragments that recur in every response from that provider, in the manner of a zstandard custom dictionary. Payload content matching an entry is stored as a reference, and a payload that mostly consists of a known envelope stores as that base plus a patch of its varying keys. Entries are append-only: stored payloads resolve against them by key at read time. The OpenAI plugin declares dictionaries for both its API shapes - on a sample of real logged turns these took total payload storage from 22% saved with string matching alone to 51%. [#1586](https://github.com/simonw/llm/pull/1586) +- Fixed streamed OpenAI Responses API calls recording two different ciphertexts of the same reasoning: the part's `encrypted_content` was harvested from the `response.output_item.done` event while `response_json` came from `response.completed`, and OpenAI encrypts per event. Reasoning metadata is now re-emitted from the final payload, so both records agree on one blob. [#1586](https://github.com/simonw/llm/pull/1586) - `llm tools` now provides improved output for dynamic toolboxes - toolbox classes such as `MCP` from [llm-mcp-client](https://github.com/simonw/llm-mcp-client) that generate their tools at runtime. These were previously listed as just their name. They are now listed with their constructor signature and class docstring. Passing one or more specifications such as `llm tools 'MCP("https://datasette.simonwillison.net/-/mcp")'` instantiates each toolbox and lists the tools that configured instance provides. `llm tools --json` output now includes a `"dynamic"` boolean key for each toolbox. [#1580](https://github.com/simonw/llm/issues/1580) (v0_32_rc2)= @@ -31,7 +34,7 @@ You can create a backup of your logs database prior to upgrading using: - `llm prompt` logs are now written to a new set of tables, not the legacy tables. - New documented {ref}`Response.log_to_db() ` Python API for writing to a SQLite logs database. - `llm logs` now reads both generations of log tables: history recorded by older versions of LLM in the legacy `responses` table is merged into the output alongside new turns, with rows from the dual-write era deduplicated by id. The `-m`, `-c`, `-f`, `-T`, `--tools` and `--schema` filters work across both. [#1562](https://github.com/simonw/llm/pull/1562) -- Raw provider response payloads are recorded on the new tables: the full `response.json()` dictionary is stored in a `turns.response_json` column, {ref}`condensed ` using [condense-json](https://github.com/simonw/condense-json) 1.1 so that content the database already holds is stored as references rather than copies - response text, reasoning summaries and their encrypted blobs, tool arguments and descriptions, and the JSON schema echoed back on structured-output calls, matched structurally so provider key ordering never matters. Model plugins can declare a `json_replacements` dictionary of recurring payload boilerplate, and a payload that mostly consists of a known envelope stores as that base plus a patch. `llm logs --json` shows the payload resolved back to exactly what the provider sent. This preserves data that only lives in the raw payload, such as the log probabilities returned by OpenAI models with `-o logprobs`. The old `prompt_json` column is no longer persisted - the stored message chain is the record of what was sent. [#1586](https://github.com/simonw/llm/pull/1586) +- Raw provider payloads (the old `prompt_json` and `response_json` columns) are no longer persisted - the stored message chain is the record of what was sent and returned. This means data that only ever lived in those raw payloads is no longer stored, notably the log probabilities returned by OpenAI completion models with `-o logprobs`. - `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. See {ref}`logging-search`. - `llm logs` now shows which configured toolbox instance provided each tool - in the tools list for a prompt and on each tool result, so a `SQLite_query` result records that it ran against `SQLite("mydb.db")`. - New {ref}`message_tree SQL view ` that renders each conversation thread as an indented text outline, for exploring the message store directly with SQL. [#1562](https://github.com/simonw/llm/pull/1562) From bafecf6a08149b0469fb587faf9476b04cb7245c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 08:59:12 -0700 Subject: [PATCH 210/258] Document json_replacements for plugin authors The mechanism was described in the message store docs and the changelog but not where plugin authors look. New section in advanced-model-plugins.md covering what to declare, with the rules that matter: entries are append-only because stored payloads resolve against them by key; prefer container entries since strings match as substrings inside model output; and reading requires the plugin to be installed. The changelog entry and the message store section both link to it. Co-Authored-By: Claude Fable 5 --- docs/changelog.md | 2 +- docs/logging.md | 2 +- docs/plugins/advanced-model-plugins.md | 36 ++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index dde8845bb..b139f0173 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -4,7 +4,7 @@ ## Unreleased - Raw provider response payloads are persisted again: the full `response.json()` dictionary is stored in a new `turns.response_json` column, {ref}`condensed ` using [condense-json](https://github.com/simonw/condense-json) 1.1 so that content the database already holds is stored as references rather than copies - response text, reasoning summaries and their encrypted blobs, tool arguments and descriptions, and the JSON schema echoed back on structured-output calls, matched structurally so provider key ordering never matters. `llm logs --json` shows the payload resolved back to exactly what the provider sent, and `LogStore.turn_response_json(turn_id)` returns it from Python. This restores data that only lives in the raw payload, such as the log probabilities returned by OpenAI models with `-o logprobs`. `prompt_json` remains unpersisted - the stored message chain is the record of what was sent. [#1586](https://github.com/simonw/llm/pull/1586) -- Model plugins can declare a `json_replacements` class attribute: a dictionary of payload fragments that recur in every response from that provider, in the manner of a zstandard custom dictionary. Payload content matching an entry is stored as a reference, and a payload that mostly consists of a known envelope stores as that base plus a patch of its varying keys. Entries are append-only: stored payloads resolve against them by key at read time. The OpenAI plugin declares dictionaries for both its API shapes - on a sample of real logged turns these took total payload storage from 22% saved with string matching alone to 51%. [#1586](https://github.com/simonw/llm/pull/1586) +- Model plugins can declare a {ref}`json_replacements ` class attribute: a dictionary of payload fragments that recur in every response from that provider, in the manner of a zstandard custom dictionary. Payload content matching an entry is stored as a reference, and a payload that mostly consists of a known envelope stores as that base plus a patch of its varying keys. Entries are append-only: stored payloads resolve against them by key at read time. The OpenAI plugin declares dictionaries for both its API shapes - on a sample of real logged turns these took total payload storage from 22% saved with string matching alone to 51%. [#1586](https://github.com/simonw/llm/pull/1586) - Fixed streamed OpenAI Responses API calls recording two different ciphertexts of the same reasoning: the part's `encrypted_content` was harvested from the `response.output_item.done` event while `response_json` came from `response.completed`, and OpenAI encrypts per event. Reasoning metadata is now re-emitted from the final payload, so both records agree on one blob. [#1586](https://github.com/simonw/llm/pull/1586) - `llm tools` now provides improved output for dynamic toolboxes - toolbox classes such as `MCP` from [llm-mcp-client](https://github.com/simonw/llm-mcp-client) that generate their tools at runtime. These were previously listed as just their name. They are now listed with their constructor signature and class docstring. Passing one or more specifications such as `llm tools 'MCP("https://datasette.simonwillison.net/-/mcp")'` instantiates each toolbox and lists the tools that configured instance provides. `llm tools --json` output now includes a `"dynamic"` boolean key for each toolbox. [#1580](https://github.com/simonw/llm/issues/1580) diff --git a/docs/logging.md b/docs/logging.md index 554fab338..e5ee2c0d0 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -538,7 +538,7 @@ The parts of a response are a normalized view of what the provider returned - en It is stored *condensed*, using [condense-json](https://github.com/simonw/condense-json). Any string of 64 characters or more that also appears in the turn's own messages - the response text, a reasoning summary and its encrypted counterpart, long tool arguments - is replaced with a reference such as `{"$": "0.0.text"}`, keyed by the message offset, part position and field the string came from. Tool definitions get the same treatment: providers echo every tool back in the payload on every call, so a long tool description is stored once in the `tools` table and referenced as `{"$": "tool.NAME.description"}`. The replacement mapping is never stored: it is rebuilt whenever the payload is read, from the stored messages (frozen by their hashes) and the turn's `turn_tools` rows. What remains in the column is roughly the provider envelope, at a fraction of the size of the raw payload and without a second copy of anything the store already holds. -Model plugins can contribute a *dictionary* of their own recurring boilerplate - the same idea as a zstandard custom dictionary - by defining a `json_replacements` class attribute: a dict mapping keys to payload fragments the plugin author knows appear verbatim in every reply, such as OpenAI's all-zero `tool_usage` accounting block. These are matched like any other replacement (dicts and lists structurally) and resolved by looking the model up again at read time. Two consequences follow: reading such payloads requires the model's plugin to be installed, and entries are **append-only** - never remove or change an existing entry, only add new ones, because editing an entry silently breaks every payload already condensed against it. +Model plugins can contribute a *dictionary* of their own recurring boilerplate - the same idea as a zstandard custom dictionary - by defining a `json_replacements` class attribute: a dict mapping keys to payload fragments the plugin author knows appear verbatim in every reply, such as OpenAI's all-zero `tool_usage` accounting block. These are matched like any other replacement (dicts and lists structurally) and resolved by looking the model up again at read time. Two consequences follow: reading such payloads requires the model's plugin to be installed, and entries are **append-only** - never remove or change an existing entry, only add new ones, because editing an entry silently breaks every payload already condensed against it. See {ref}`the plugin author documentation ` for guidance on declaring these. `llm logs --json` resolves the stored payload back to exactly what the provider sent and includes it as `response_json`. From Python, `LogStore.turn_response_json(turn_id)` returns the same resolved dictionary. Turns logged before this column existed, and models that expose no raw payload, record NULL. diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index 4bf814d3a..1d10c966f 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -512,6 +512,42 @@ else: ) ``` +(advanced-model-plugins-json-replacements)= + +## Condensing logged payloads with json_replacements + +The raw provider payload your plugin assigns to `response.response_json` is {ref}`logged condensed `: content that the log database already stores elsewhere - the response text, reasoning blobs, tool definitions - is replaced with references instead of being written twice. + +Your model class can improve on this by declaring a `json_replacements` class attribute: a dictionary of payload fragments that you know recur verbatim in every response from your provider, in the manner of a zstandard custom dictionary. Typical entries are all-zero usage accounting blocks and echoed default settings: + +```python +from typing import ClassVar + + +class MyModel(llm.KeyModel): + json_replacements: ClassVar[dict] = { + "tool_usage_0": { + "image_gen": {"input_tokens": 0, "output_tokens": 0}, + "web_search": {"num_requests": 0}, + }, + "response_env_0": { + "object": "response", + "status": "completed", + "store": False, + }, + } +``` + +Payload content matching an entry is stored as a reference to it. Dict entries match structurally - the provider's key ordering never matters - and additionally serve as *merge bases*: a payload object that mostly matches an entry, such as a response envelope where only the id and usage vary per call, is stored as the base plus a patch of its differing keys. + +Rules to follow: + +- **Entries are append-only. Never remove or change an existing entry - only add new ones.** Stored payloads reference entries by key and resolve against your dictionary when they are read back, so editing an entry silently breaks every payload already logged against it. When a provider changes a boilerplate block, append a new entry with a new suffix (`tool_usage_1`) and leave the old one in place. +- Prefer dict and list entries. String entries match as substrings anywhere in the payload, including inside the model's text output, so short generic strings cause wasteful reference churn. +- Reading these payloads requires your plugin to be installed: resolution looks your model up by id in the registry. If the model is unknown at read time, the payload is reported as unavailable rather than resolved incorrectly. + +There is no size threshold for dictionary entries - you are trusted to curate them - but an entry smaller than the roughly 20-byte reference that replaces it makes payloads larger, not smaller. + ## Consuming prompt.messages in build_messages `prompt.messages` is an `list[llm.Message]` that is always **the complete input chain for this turn** — whether the caller supplied it explicitly via `model.prompt(messages=[...])`, or it was synthesized from kwargs (`prompt=`, `system=`, `attachments=`, `tool_results=`), or it was pre-built by a `Conversation` or by `response.reply()`. From f84f2836a1da32cdf9ba87702c9d356f0f5c4e27 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 09:00:20 -0700 Subject: [PATCH 211/258] Ran cog --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ec6f904ac..a851e1f99 100644 --- a/README.md +++ b/README.md @@ -299,6 +299,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Supporting tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-tools) * [Attachments for multi-modal models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#attachments-for-multi-modal-models) * [Structured messages and streaming events](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#structured-messages-and-streaming-events) + * [Condensing logged payloads with json_replacements](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#condensing-logged-payloads-with-json-replacements) * [Consuming prompt.messages in build_messages](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#consuming-prompt-messages-in-build-messages) * [Restoring opaque metadata on subsequent requests](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#restoring-opaque-metadata-on-subsequent-requests) * [Tracking token usage](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-token-usage) From 9a225319ea5b6fa37c14236e29e93777f4c4e56f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 12:02:45 -0700 Subject: [PATCH 212/258] Copy edit documentation and changelog --- docs/changelog.md | 6 +++--- docs/logging.md | 8 ++++---- docs/plugins/advanced-model-plugins.md | 15 +++++---------- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index b139f0173..8bb1687fb 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,10 +3,10 @@ (unreleased)= ## Unreleased -- Raw provider response payloads are persisted again: the full `response.json()` dictionary is stored in a new `turns.response_json` column, {ref}`condensed ` using [condense-json](https://github.com/simonw/condense-json) 1.1 so that content the database already holds is stored as references rather than copies - response text, reasoning summaries and their encrypted blobs, tool arguments and descriptions, and the JSON schema echoed back on structured-output calls, matched structurally so provider key ordering never matters. `llm logs --json` shows the payload resolved back to exactly what the provider sent, and `LogStore.turn_response_json(turn_id)` returns it from Python. This restores data that only lives in the raw payload, such as the log probabilities returned by OpenAI models with `-o logprobs`. `prompt_json` remains unpersisted - the stored message chain is the record of what was sent. [#1586](https://github.com/simonw/llm/pull/1586) -- Model plugins can declare a {ref}`json_replacements ` class attribute: a dictionary of payload fragments that recur in every response from that provider, in the manner of a zstandard custom dictionary. Payload content matching an entry is stored as a reference, and a payload that mostly consists of a known envelope stores as that base plus a patch of its varying keys. Entries are append-only: stored payloads resolve against them by key at read time. The OpenAI plugin declares dictionaries for both its API shapes - on a sample of real logged turns these took total payload storage from 22% saved with string matching alone to 51%. [#1586](https://github.com/simonw/llm/pull/1586) -- Fixed streamed OpenAI Responses API calls recording two different ciphertexts of the same reasoning: the part's `encrypted_content` was harvested from the `response.output_item.done` event while `response_json` came from `response.completed`, and OpenAI encrypts per event. Reasoning metadata is now re-emitted from the final payload, so both records agree on one blob. [#1586](https://github.com/simonw/llm/pull/1586) - `llm tools` now provides improved output for dynamic toolboxes - toolbox classes such as `MCP` from [llm-mcp-client](https://github.com/simonw/llm-mcp-client) that generate their tools at runtime. These were previously listed as just their name. They are now listed with their constructor signature and class docstring. Passing one or more specifications such as `llm tools 'MCP("https://datasette.simonwillison.net/-/mcp")'` instantiates each toolbox and lists the tools that configured instance provides. `llm tools --json` output now includes a `"dynamic"` boolean key for each toolbox. [#1580](https://github.com/simonw/llm/issues/1580) +- Raw provider response payloads are persisted again: the full `response.json()` dictionary is stored in a new `turns.response_json` column, {ref}`condensed ` using [condense-json](https://github.com/simonw/condense-json) 1.1. `llm logs --json` shows the payload resolved back to what the provider sent, and `LogStore.turn_response_json(turn_id)` returns it from Python. This restores data that only lives in the raw payload, such as the log probabilities returned by OpenAI models with `-o logprobs`. [#1586](https://github.com/simonw/llm/pull/1586) +- Model plugins can declare a {ref}`json_replacements ` class attribute to further reduce the size of the condendsed JSON. The OpenAI plugin declares dictionaries for both its API shapes - on a sample of real logged turns these took total payload storage from 22% saved with string matching alone to 51%. +- Fixed streamed OpenAI Responses API calls recording two different ciphertexts of the same reasoning: the part's `encrypted_content` was harvested from the `response.output_item.done` event while `response_json` came from `response.completed`, and OpenAI encrypts per event. Reasoning metadata is now re-emitted from the final payload, so both records agree on one blob. (v0_32_rc2)= ## 0.32rc2 (2026-07-30) diff --git a/docs/logging.md b/docs/logging.md index e5ee2c0d0..ac4c16088 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -534,13 +534,13 @@ Attachments work the same way: the binary content lives in the `attachments` tab ### The raw provider payload -The parts of a response are a normalized view of what the provider returned - enough to replay the conversation, but not the whole story. The raw `response.json()` dictionary is also recorded on the turn, in the `turns.response_json` column, so the details that have no part equivalent - usage breakdowns, system fingerprints, logprobs, settings echoes - survive too. +The parts of a response are a normalized view of what the provider returned. The raw `response.json()` dictionary is also recorded in the `turns.response_json` column, storing details that have no part equivalent - usage breakdowns, system fingerprints, logprobs, and more. -It is stored *condensed*, using [condense-json](https://github.com/simonw/condense-json). Any string of 64 characters or more that also appears in the turn's own messages - the response text, a reasoning summary and its encrypted counterpart, long tool arguments - is replaced with a reference such as `{"$": "0.0.text"}`, keyed by the message offset, part position and field the string came from. Tool definitions get the same treatment: providers echo every tool back in the payload on every call, so a long tool description is stored once in the `tools` table and referenced as `{"$": "tool.NAME.description"}`. The replacement mapping is never stored: it is rebuilt whenever the payload is read, from the stored messages (frozen by their hashes) and the turn's `turn_tools` rows. What remains in the column is roughly the provider envelope, at a fraction of the size of the raw payload and without a second copy of anything the store already holds. +The JSON is in a condensed form using [condense-json](https://github.com/simonw/condense-json). This library allows values that are stored in the other database rows to be replaced in the JSON as special references such as `{"$": "tool.NAME.description"}`. -Model plugins can contribute a *dictionary* of their own recurring boilerplate - the same idea as a zstandard custom dictionary - by defining a `json_replacements` class attribute: a dict mapping keys to payload fragments the plugin author knows appear verbatim in every reply, such as OpenAI's all-zero `tool_usage` accounting block. These are matched like any other replacement (dicts and lists structurally) and resolved by looking the model up again at read time. Two consequences follow: reading such payloads requires the model's plugin to be installed, and entries are **append-only** - never remove or change an existing entry, only add new ones, because editing an entry silently breaks every payload already condensed against it. See {ref}`the plugin author documentation ` for guidance on declaring these. +Model plugins can contribute a *dictionary* of their own recurring boilerplate - the same idea as a zstandard custom dictionary - by defining a `json_replacements` class attribute. See {ref}`the plugin author documentation ` for guidance on declaring these. -`llm logs --json` resolves the stored payload back to exactly what the provider sent and includes it as `response_json`. From Python, `LogStore.turn_response_json(turn_id)` returns the same resolved dictionary. Turns logged before this column existed, and models that expose no raw payload, record NULL. +`llm logs --json` resolves the stored payload back to the original sen tby the provider as the `response_json` (logging-message-store-tables)= diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index 1d10c966f..fe494d88e 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -518,14 +518,11 @@ else: The raw provider payload your plugin assigns to `response.response_json` is {ref}`logged condensed `: content that the log database already stores elsewhere - the response text, reasoning blobs, tool definitions - is replaced with references instead of being written twice. -Your model class can improve on this by declaring a `json_replacements` class attribute: a dictionary of payload fragments that you know recur verbatim in every response from your provider, in the manner of a zstandard custom dictionary. Typical entries are all-zero usage accounting blocks and echoed default settings: +Your model class can improve on this by declaring a `json_replacements` class attribute: a dictionary of payload fragments that you know recur in every response from your provider. Consult [the default OpenAI plugin](https://github.com/simonw/llm/blob/main/llm/default_plugins/openai_models.py) for an example of this pattern. ```python -from typing import ClassVar - - class MyModel(llm.KeyModel): - json_replacements: ClassVar[dict] = { + json_replacements = { "tool_usage_0": { "image_gen": {"input_tokens": 0, "output_tokens": 0}, "web_search": {"num_requests": 0}, @@ -538,15 +535,13 @@ class MyModel(llm.KeyModel): } ``` -Payload content matching an entry is stored as a reference to it. Dict entries match structurally - the provider's key ordering never matters - and additionally serve as *merge bases*: a payload object that mostly matches an entry, such as a response envelope where only the id and usage vary per call, is stored as the base plus a patch of its differing keys. +Payload content matching an entry is stored as a reference to it. Dict entries match structurally and additionally serve as *merge bases*: a payload object that mostly matches an entry, such as a response envelope where only the id and usage vary per call, is stored as the base plus a patch of its differing keys. Rules to follow: -- **Entries are append-only. Never remove or change an existing entry - only add new ones.** Stored payloads reference entries by key and resolve against your dictionary when they are read back, so editing an entry silently breaks every payload already logged against it. When a provider changes a boilerplate block, append a new entry with a new suffix (`tool_usage_1`) and leave the old one in place. -- Prefer dict and list entries. String entries match as substrings anywhere in the payload, including inside the model's text output, so short generic strings cause wasteful reference churn. +- **Entries are append-only. Never remove or change an existing entry - only add new ones.** Stored payloads reference entries by key and resolve against your dictionary when they are read back, so editing an entry silently breaks every payload already logged against it. When a provider changes a boilerplate block, append a new entry with a new suffix and leave the old one in place. - Reading these payloads requires your plugin to be installed: resolution looks your model up by id in the registry. If the model is unknown at read time, the payload is reported as unavailable rather than resolved incorrectly. - -There is no size threshold for dictionary entries - you are trusted to curate them - but an entry smaller than the roughly 20-byte reference that replaces it makes payloads larger, not smaller. +- There is no size threshold for dictionary entries - you are trusted to curate them - but an entry smaller than the roughly 20-byte reference that replaces it makes payloads larger, not smaller. ## Consuming prompt.messages in build_messages From de680408b464e5f8cee4f4bcb4c1b7d3f51bd75c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 12:05:31 -0700 Subject: [PATCH 213/258] Trimmed some long comments --- llm/default_plugins/openai_models.py | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index e444e2edf..6214e3563 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1008,16 +1008,8 @@ def _attachment(attachment, image_detail=None): class _Shared: - # Boilerplate fragments that recur verbatim in this API's response - # payloads, offered to the log store's payload condensing as shared - # replacements - the same idea as a zstandard custom dictionary. - # Dict and list values match payload subtrees structurally, so key - # order does not matter. - # # NEVER remove or change an existing entry - only ever append new - # ones. Stored payloads reference these by key and resolve against - # the dictionary at read time, so editing an entry silently breaks - # every payload already condensed against it. + # ones. json_replacements: ClassVar[dict] = { "completion_tokens_details_0": { "accepted_prediction_tokens": 0, @@ -1521,8 +1513,7 @@ class _SharedResponses(_Shared): # Recurring boilerplate in Responses API payloads. Same contract as # _Shared.json_replacements, which this replaces for Responses # models: NEVER remove or change an existing entry - only ever - # append new ones, or payloads already stored against these keys - # stop resolving. + # append new ones. json_replacements: ClassVar[dict] = { "tool_usage_0": { "image_gen": { @@ -1544,9 +1535,6 @@ class _SharedResponses(_Shared): "cached_tokens": 0, "cache_write_tokens": 0, }, - # The reasoning settings echo, in the variants the API emits - # depending on reasoning context. New variants get appended as - # reasoning_settings_2 and so on - never edited in place. "reasoning_settings_0": { "effort": "medium", "summary": "detailed", @@ -1561,11 +1549,7 @@ class _SharedResponses(_Shared): }, # The default text block on non-schema replies "text_format_0": {"format": {"type": "text"}, "verbosity": "medium"}, - # The static envelope of a Responses payload. Dict entries also - # act as condense-json merge bases: a payload top-level that - # differs from this in a few keys stores as this base plus a - # patch, so settings a config changes (top_p, say) simply ride - # in the patch. + # The static envelope of a Responses payload "response_env_0": { "object": "response", "parallel_tool_calls": True, From 2934441e661f39d373a3352476d3032411b17cab Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 12:38:24 -0700 Subject: [PATCH 214/258] service_tier option for OpenAI models PR #1585 https://claude.ai/code/session_01CYbKrezWv4ANYuXGFjJGkN --- README.md | 1 + docs/openai-models.md | 31 ++++++ docs/usage.md | 41 ++++++++ llm/default_plugins/openai_models.py | 142 +++++++++++++++++++++++---- tests/test_openai_responses.py | 103 +++++++++++++++++++ 5 files changed, 300 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a851e1f99..9b7dd1eb3 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Configuration](https://llm.datasette.io/en/stable/openai-models.html#configuration) * [OpenAI language models](https://llm.datasette.io/en/stable/openai-models.html#openai-language-models) * [Model features](https://llm.datasette.io/en/stable/openai-models.html#model-features) + * [Fast mode and service tiers](https://llm.datasette.io/en/stable/openai-models.html#fast-mode-and-service-tiers) * [OpenAI embedding models](https://llm.datasette.io/en/stable/openai-models.html#openai-embedding-models) * [OpenAI completion models](https://llm.datasette.io/en/stable/openai-models.html#openai-completion-models) * [Adding more OpenAI models](https://llm.datasette.io/en/stable/openai-models.html#adding-more-openai-models) diff --git a/docs/openai-models.md b/docs/openai-models.md index 31f54e3c1..ddb68ff92 100644 --- a/docs/openai-models.md +++ b/docs/openai-models.md @@ -83,6 +83,35 @@ The following features work with OpenAI models: - {ref}`Schemas ` can be used to influence the JSON structure of the model output. - {ref}`Model options ` can be used to set parameters like `temperature`. Use `llm models --options` for a full list of supported options. +(openai-models-service-tier)= + +## Fast mode and service tiers + +OpenAI models can process requests at different speeds and prices using the `service_tier` API parameter. [Fast mode](https://developers.openai.com/api/docs/guides/fast-mode) runs up to 2.5x faster than standard processing at a higher per-token price, with the biggest speed increase on `gpt-5.6-sol`. [Flex processing](https://developers.openai.com/api/docs/guides/flex-processing) is slower but cheaper. Each tier works with a different subset of models - the Fast and Flex tables on [OpenAI's pricing page](https://developers.openai.com/api/docs/pricing) are the definitive list of which models support which tier. + +All of the OpenAI models supported by LLM expose a `service_tier` option, with the exception of the legacy `gpt-3.5-turbo-instruct` completion model. Use `-o service_tier fast` to enable Fast mode for a prompt: + +```bash +llm -m gpt-5.6-sol -o service_tier fast 'Fast facts about pelicans' +``` + +The value is passed straight to the API, so other tiers such as `priority` (the older name for `fast`) and `flex` work too: + +```bash +llm -m gpt-5.4 -o service_tier flex 'No rush: facts about pelicans' +``` + +The requested tier is recorded in the logged options for the prompt, visible in `llm logs -c --json`. The API response also reports the service tier that actually processed the request - OpenAI may fall back to standard processing if Fast mode capacity is unavailable. Using the {ref}`Python API ` you can check that with: + +```python +import llm + +model = llm.get_model("gpt-5.6-sol") +response = model.prompt("Fast facts about pelicans", service_tier="fast") +print(response.text()) +print(response.json()["service_tier"]) +``` + (openai-models-embedding)= ## OpenAI embedding models @@ -155,6 +184,8 @@ If the model supports structured extraction using json_schema, add `supports_sch For reasoning models like `o1` or `o3-mini` add `reasoning: true`. +If the model supports the `service_tier` parameter - see {ref}`openai-models-service-tier` - add `service_tier: true` to enable the corresponding option. + With this configuration in place, the following command should run a prompt against the new model: ```bash diff --git a/docs/usage.md b/docs/usage.md index 72af52963..ff55e5d03 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -629,6 +629,10 @@ OpenAI Chat: gpt-4o (aliases: 4o) image_detail: str Controls the detail level for image attachments. Supported values are low, high, and auto. + service_tier: str + The processing tier to use for this request - for example 'fast' for + Fast mode (faster responses at a higher price) or 'flex' for slower, + cheaper processing on models that support those tiers. Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -651,6 +655,7 @@ OpenAI Chat: gpt-4o-mini (aliases: 4o-mini) seed: int json_object: boolean image_detail: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -673,6 +678,7 @@ OpenAI Chat: gpt-4.1 (aliases: 4.1) seed: int json_object: boolean image_detail: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -695,6 +701,7 @@ OpenAI Chat: gpt-4.1-mini (aliases: 4.1-mini) seed: int json_object: boolean image_detail: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -717,6 +724,7 @@ OpenAI Chat: gpt-4.1-nano (aliases: 4.1-nano) seed: int json_object: boolean image_detail: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -739,6 +747,7 @@ OpenAI Chat: gpt-3.5-turbo (aliases: 3.5, chatgpt) seed: int json_object: boolean image_detail: str + service_tier: str Features: - streaming - async @@ -757,6 +766,7 @@ OpenAI Chat: gpt-3.5-turbo-16k (aliases: chatgpt-16k, 3.5-16k) seed: int json_object: boolean image_detail: str + service_tier: str Features: - streaming - async @@ -775,6 +785,7 @@ OpenAI Chat: gpt-4 (aliases: 4, gpt4) seed: int json_object: boolean image_detail: str + service_tier: str Features: - streaming - async @@ -793,6 +804,7 @@ OpenAI Chat: gpt-4-turbo-2024-04-09 seed: int json_object: boolean image_detail: str + service_tier: str Features: - streaming - async @@ -811,6 +823,7 @@ OpenAI Chat: gpt-4-turbo (aliases: gpt-4-turbo-preview, 4-turbo, 4t) seed: int json_object: boolean image_detail: str + service_tier: str Features: - streaming - async @@ -860,6 +873,10 @@ OpenAI Responses: o1 supported values are low, medium, and high. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning in a response. + service_tier: str + The processing tier to use for this request - for example 'fast' for + Fast mode (faster responses at a higher price) or 'flex' for slower, + cheaper processing on models that support those tiers. Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -883,6 +900,7 @@ OpenAI Responses: o1-2024-12-17 chat_completions: boolean image_detail: str reasoning_effort: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -906,6 +924,7 @@ OpenAI Responses: o3-mini chat_completions: boolean image_detail: str reasoning_effort: str + service_tier: str Features: - streaming - schemas @@ -928,6 +947,7 @@ OpenAI Responses: o3 chat_completions: boolean image_detail: str reasoning_effort: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -952,6 +972,7 @@ OpenAI Responses: o4-mini chat_completions: boolean image_detail: str reasoning_effort: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -977,6 +998,7 @@ OpenAI Responses: gpt-5 image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1002,6 +1024,7 @@ OpenAI Responses: gpt-5-mini image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1027,6 +1050,7 @@ OpenAI Responses: gpt-5-nano image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1052,6 +1076,7 @@ OpenAI Responses: gpt-5-2025-08-07 image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1077,6 +1102,7 @@ OpenAI Responses: gpt-5-mini-2025-08-07 image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1102,6 +1128,7 @@ OpenAI Responses: gpt-5-nano-2025-08-07 image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1127,6 +1154,7 @@ OpenAI Responses: gpt-5.1 image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1152,6 +1180,7 @@ OpenAI Responses: gpt-5.2 image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1177,6 +1206,7 @@ OpenAI Responses: gpt-5.2-chat-latest image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1202,6 +1232,7 @@ OpenAI Responses: gpt-5.4 image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1227,6 +1258,7 @@ OpenAI Responses: gpt-5.4-2026-03-05 image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1252,6 +1284,7 @@ OpenAI Responses: gpt-5.4-mini image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1277,6 +1310,7 @@ OpenAI Responses: gpt-5.4-mini-2026-03-17 image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1302,6 +1336,7 @@ OpenAI Responses: gpt-5.4-nano image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1327,6 +1362,7 @@ OpenAI Responses: gpt-5.4-nano-2026-03-17 image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1352,6 +1388,7 @@ OpenAI Responses: gpt-5.5 image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1377,6 +1414,7 @@ OpenAI Responses: gpt-5.5-2026-04-23 image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1402,6 +1440,7 @@ OpenAI Responses: gpt-5.6-sol image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1427,6 +1466,7 @@ OpenAI Responses: gpt-5.6-terra image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: @@ -1452,6 +1492,7 @@ OpenAI Responses: gpt-5.6-luna image_detail: str reasoning_effort: str verbosity: str + service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp Features: diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 6214e3563..fb589dcdd 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -38,39 +38,82 @@ def register_models(register): # GPT-4o register( - Chat("gpt-4o", vision=True, supports_schema=True, supports_tools=True), - AsyncChat("gpt-4o", vision=True, supports_schema=True, supports_tools=True), + Chat( + "gpt-4o", + vision=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), + AsyncChat( + "gpt-4o", + vision=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), aliases=("4o",), ) register( - Chat("gpt-4o-mini", vision=True, supports_schema=True, supports_tools=True), + Chat( + "gpt-4o-mini", + vision=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), AsyncChat( - "gpt-4o-mini", vision=True, supports_schema=True, supports_tools=True + "gpt-4o-mini", + vision=True, + service_tier=True, + supports_schema=True, + supports_tools=True, ), aliases=("4o-mini",), ) # GPT-4.1 for model_id in ("gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano"): register( - Chat(model_id, vision=True, supports_schema=True, supports_tools=True), - AsyncChat(model_id, vision=True, supports_schema=True, supports_tools=True), + Chat( + model_id, + vision=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), + AsyncChat( + model_id, + vision=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), aliases=(model_id.replace("gpt-", ""),), ) # 3.5 and 4 register( - Chat("gpt-3.5-turbo"), AsyncChat("gpt-3.5-turbo"), aliases=("3.5", "chatgpt") + Chat("gpt-3.5-turbo", service_tier=True), + AsyncChat("gpt-3.5-turbo", service_tier=True), + aliases=("3.5", "chatgpt"), ) register( - Chat("gpt-3.5-turbo-16k"), - AsyncChat("gpt-3.5-turbo-16k"), + Chat("gpt-3.5-turbo-16k", service_tier=True), + AsyncChat("gpt-3.5-turbo-16k", service_tier=True), aliases=("chatgpt-16k", "3.5-16k"), ) - register(Chat("gpt-4"), AsyncChat("gpt-4"), aliases=("4", "gpt4")) + register( + Chat("gpt-4", service_tier=True), + AsyncChat("gpt-4", service_tier=True), + aliases=("4", "gpt4"), + ) # GPT-4 Turbo models - register(Chat("gpt-4-turbo-2024-04-09"), AsyncChat("gpt-4-turbo-2024-04-09")) register( - Chat("gpt-4-turbo"), - AsyncChat("gpt-4-turbo"), + Chat("gpt-4-turbo-2024-04-09", service_tier=True), + AsyncChat("gpt-4-turbo-2024-04-09", service_tier=True), + ) + register( + Chat("gpt-4-turbo", service_tier=True), + AsyncChat("gpt-4-turbo", service_tier=True), aliases=("gpt-4-turbo-preview", "4-turbo", "4t"), ) # o1 @@ -81,6 +124,7 @@ def register_models(register): vision=True, can_stream=False, reasoning=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -89,23 +133,44 @@ def register_models(register): vision=True, can_stream=False, reasoning=True, + service_tier=True, supports_schema=True, supports_tools=True, ), ) register( - Responses("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), + Responses( + "o3-mini", + reasoning=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), AsyncResponses( - "o3-mini", reasoning=True, supports_schema=True, supports_tools=True + "o3-mini", + reasoning=True, + service_tier=True, + supports_schema=True, + supports_tools=True, ), ) register( Responses( - "o3", vision=True, reasoning=True, supports_schema=True, supports_tools=True + "o3", + vision=True, + reasoning=True, + service_tier=True, + supports_schema=True, + supports_tools=True, ), AsyncResponses( - "o3", vision=True, reasoning=True, supports_schema=True, supports_tools=True + "o3", + vision=True, + reasoning=True, + service_tier=True, + supports_schema=True, + supports_tools=True, ), ) register( @@ -113,6 +178,7 @@ def register_models(register): "o4-mini", vision=True, reasoning=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -120,6 +186,7 @@ def register_models(register): "o4-mini", vision=True, reasoning=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -139,6 +206,7 @@ def register_models(register): vision=True, reasoning=True, verbosity=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -147,6 +215,7 @@ def register_models(register): vision=True, reasoning=True, verbosity=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -158,6 +227,7 @@ def register_models(register): vision=True, reasoning=True, verbosity=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -166,6 +236,7 @@ def register_models(register): vision=True, reasoning=True, verbosity=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -178,6 +249,7 @@ def register_models(register): vision=True, reasoning=True, verbosity=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -186,6 +258,7 @@ def register_models(register): vision=True, reasoning=True, verbosity=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -208,6 +281,7 @@ def register_models(register): reasoning=True, verbosity=True, image_detail_original=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -217,6 +291,7 @@ def register_models(register): reasoning=True, verbosity=True, image_detail_original=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -234,6 +309,7 @@ def register_models(register): reasoning=True, verbosity=True, image_detail_original=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -243,6 +319,7 @@ def register_models(register): reasoning=True, verbosity=True, image_detail_original=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -257,6 +334,7 @@ def register_models(register): reasoning=True, verbosity=True, image_detail_original=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -266,6 +344,7 @@ def register_models(register): reasoning=True, verbosity=True, image_detail_original=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -304,6 +383,8 @@ def register_models(register): kwargs["vision"] = True if extra_model.get("audio") is True: kwargs["audio"] = True + if extra_model.get("service_tier") is True: + kwargs["service_tier"] = True if extra_model.get("completion"): klass = Completion async_klass = None @@ -911,6 +992,7 @@ def build_options_class( verbosity=False, image_detail_original=False, chat_completions=False, + service_tier=False, ): fields = { "json_object": ( @@ -972,6 +1054,19 @@ def build_options_class( default=None, ), ) + if service_tier: + fields["service_tier"] = ( + str | None, + Field( + description=( + "The processing tier to use for this request - for example " + "'fast' for Fast mode (faster responses at a higher price) " + "or 'flex' for slower, cheaper processing on models that " + "support those tiers." + ), + default=None, + ), + ) return create_model("Options", __base__=SharedOptions, **fields) @@ -1039,6 +1134,7 @@ def __init__( reasoning=False, verbosity=False, image_detail_original=False, + service_tier=False, supports_schema=False, supports_tools=False, allows_system_prompt=True, @@ -1059,11 +1155,12 @@ def __init__( self.attachment_types = set() - if reasoning or verbosity or image_detail_original: + if reasoning or verbosity or image_detail_original or service_tier: self.Options = build_options_class( reasoning=reasoning, verbosity=verbosity, image_detail_original=image_detail_original, + service_tier=service_tier, ) if vision: @@ -1595,6 +1692,7 @@ def _delegate_chat_kwargs(self): "reasoning": self._reasoning, "verbosity": self._verbosity, "image_detail_original": self._image_detail_original, + "service_tier": self._service_tier, "supports_schema": self.supports_schema, "supports_tools": self.supports_tools, "allows_system_prompt": self.allows_system_prompt, @@ -1891,6 +1989,7 @@ def __init__( reasoning=False, verbosity=False, image_detail_original=False, + service_tier=False, supports_schema=False, supports_tools=False, allows_system_prompt=True, @@ -1911,6 +2010,7 @@ def __init__( reasoning=reasoning, verbosity=verbosity, image_detail_original=image_detail_original, + service_tier=service_tier, supports_schema=supports_schema, supports_tools=supports_tools, allows_system_prompt=allows_system_prompt, @@ -1919,6 +2019,7 @@ def __init__( self._reasoning_summary = reasoning_summary self._verbosity = verbosity self._image_detail_original = image_detail_original + self._service_tier = service_tier # Override the Options class so that ``-o chat_completions 1`` is # always available on Responses-routed models. self.Options = build_options_class( @@ -1926,6 +2027,7 @@ def __init__( verbosity=verbosity, image_detail_original=image_detail_original, chat_completions=True, + service_tier=service_tier, ) def execute( @@ -2134,6 +2236,7 @@ def __init__( reasoning=False, verbosity=False, image_detail_original=False, + service_tier=False, supports_schema=False, supports_tools=False, allows_system_prompt=True, @@ -2154,6 +2257,7 @@ def __init__( reasoning=reasoning, verbosity=verbosity, image_detail_original=image_detail_original, + service_tier=service_tier, supports_schema=supports_schema, supports_tools=supports_tools, allows_system_prompt=allows_system_prompt, @@ -2162,11 +2266,13 @@ def __init__( self._reasoning_summary = reasoning_summary self._verbosity = verbosity self._image_detail_original = image_detail_original + self._service_tier = service_tier self.Options = build_options_class( reasoning=reasoning, verbosity=verbosity, image_detail_original=image_detail_original, chat_completions=True, + service_tier=service_tier, ) async def execute( diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index f7e9c6588..2dfb95ff6 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -423,6 +423,109 @@ class FakePrompt: assert "reasoning" not in kwargs +@pytest.mark.parametrize( + "model_id,expected", + [ + ("gpt-5.6-sol", True), + ("gpt-5.6-terra", True), + ("gpt-5.6-luna", True), + ("gpt-5.5", True), + ("gpt-4o", True), + # The legacy /v1/completions endpoint does not accept service_tier + ("gpt-3.5-turbo-instruct", False), + ], +) +def test_service_tier_option_on_models(model_id, expected): + model = llm.get_model(model_id) + assert ("service_tier" in model.Options.model_fields) == expected + + +def test_responses_kwargs_includes_service_tier(): + model = llm.get_model("gpt-5.6-sol") + options = model.Options(service_tier="fast") + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + kwargs = model._build_responses_kwargs(p, stream=False) + assert kwargs["service_tier"] == "fast" + + +def test_service_tier_sent_to_responses_endpoint(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_test_1", + "object": "response", + "created_at": 1, + "model": "gpt-5.6-sol", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "fast reply", + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + }, + "status": "completed", + "service_tier": "priority", + }, + headers={"Content-Type": "application/json"}, + ) + model = llm.get_model("gpt-5.6-sol") + response = model.prompt("hello", stream=False, service_tier="fast", key="test") + assert response.text() == "fast reply" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["service_tier"] == "fast" + # The response body reports the tier that actually processed the request + assert response.json()["service_tier"] == "priority" + + +def test_service_tier_sent_to_chat_completions_fallback(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "id": "chatcmpl-x", + "object": "chat.completion", + "model": "gpt-5.6-sol", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "fast chat"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + }, + headers={"Content-Type": "application/json"}, + ) + model = llm.get_model("gpt-5.6-sol") + response = model.prompt( + "hello", stream=False, chat_completions=True, service_tier="fast", key="test" + ) + assert response.text() == "fast chat" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["service_tier"] == "fast" + + def test_responses_streams_reasoning_summary_text(httpx_mock): httpx_mock.add_response( method="POST", From f7fa5b3e8967b2171f97cb56c3b9f3d9a3d8acee Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 12:40:57 -0700 Subject: [PATCH 215/258] Simplified docs for service_tier, refs #1585 --- docs/openai-models.md | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/docs/openai-models.md b/docs/openai-models.md index ddb68ff92..ed7b7f34d 100644 --- a/docs/openai-models.md +++ b/docs/openai-models.md @@ -95,23 +95,12 @@ All of the OpenAI models supported by LLM expose a `service_tier` option, with t llm -m gpt-5.6-sol -o service_tier fast 'Fast facts about pelicans' ``` -The value is passed straight to the API, so other tiers such as `priority` (the older name for `fast`) and `flex` work too: +The value is passed straight to the API, so other tiers such as `priority` (the older name for `fast`) and `flex` ([slower but cheaper processing](https://developers.openai.com/api/docs/guides/flex-processing)) work too: ```bash llm -m gpt-5.4 -o service_tier flex 'No rush: facts about pelicans' ``` -The requested tier is recorded in the logged options for the prompt, visible in `llm logs -c --json`. The API response also reports the service tier that actually processed the request - OpenAI may fall back to standard processing if Fast mode capacity is unavailable. Using the {ref}`Python API ` you can check that with: - -```python -import llm - -model = llm.get_model("gpt-5.6-sol") -response = model.prompt("Fast facts about pelicans", service_tier="fast") -print(response.text()) -print(response.json()["service_tier"]) -``` - (openai-models-embedding)= ## OpenAI embedding models From 6fb155df7a78ad6e5e5f854aab0c8380a9c5afea Mon Sep 17 00:00:00 2001 From: ikatyal2110 <134458944+ikatyal2110@users.noreply.github.com> Date: Tue, 4 Aug 2026 01:16:39 +0530 Subject: [PATCH 216/258] Fix llm openai endpoint ignoring --schema when template also defines schema_object PR #1588 --- llm/default_plugins/openai_models.py | 2 +- tests/test_openai_endpoint.py | 55 +++++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index fb589dcdd..2cb55b97e 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -665,7 +665,7 @@ def endpoint( raise click.ClickException(str(ex)) if not model_id and template_obj.model: model_id = template_obj.model - if template_obj.schema_object: + if template_obj.schema_object and not schema: schema = template_obj.schema_object if template_obj.options: options = _merge_template_options(template_obj, options) diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py index a7659c011..cf68934a2 100644 --- a/tests/test_openai_endpoint.py +++ b/tests/test_openai_endpoint.py @@ -305,15 +305,12 @@ def test_endpoint_template(httpx_mock, user_path, templates_path): }, ], "model": "template-model", + # CLI --schema takes precedence over template schema_object "response_format": { "type": "json_schema", "json_schema": { "name": "output", - "schema": { - "type": "object", - "properties": {"answer": {"type": "string"}}, - "required": ["answer"], - }, + "schema": {"type": "object"}, }, }, "stream": False, @@ -321,6 +318,54 @@ def test_endpoint_template(httpx_mock, user_path, templates_path): } +def test_endpoint_template_schema_object_used_when_no_cli_schema( + httpx_mock, user_path, templates_path +): + base_url = "https://templates2.example.test/v1" + _add_chat_response(httpx_mock, base_url, "Template response 2") + (templates_path / "schemaonly.yaml").write_text( + """ +model: schema-model +schema_object: + type: object + properties: + answer: + type: string + required: + - answer +""".strip(), + "utf-8", + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "test question", + "--template", + "schemaonly", + "--no-stream", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["response_format"] == { + "type": "json_schema", + "json_schema": { + "name": "output", + "schema": { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + }, + }, + } + + def test_endpoint_schema(httpx_mock, user_path): base_url = "https://schema.example.test/v1" _add_chat_response(httpx_mock, base_url, '{"name": "Cleo", "age": 10}') From 9a06c5685e777630d6826f819dfff89f1762805d Mon Sep 17 00:00:00 2001 From: Ojas Sharma <67553823+ojassharma7@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:13:10 -0400 Subject: [PATCH 217/258] Follow redirects (up to 3 times) on attachment URLs (#1579) Should address "Ollama OCR with URLs fails where filepaths succeed" - closes #1046 Co-authored-by: Simon Willison --- llm/models.py | 6 ++-- tests/test_attachments.py | 62 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/llm/models.py b/llm/models.py index da6fbc2ef..0a8098991 100644 --- a/llm/models.py +++ b/llm/models.py @@ -92,7 +92,8 @@ def resolve_type(self): if self.path: return mimetype_from_path(self.path) if self.url: - response = httpx.head(self.url) + with httpx.Client(follow_redirects=True, max_redirects=3) as client: + response = client.head(self.url) response.raise_for_status() return response.headers.get("content-type") if self.content: @@ -106,7 +107,8 @@ def content_bytes(self): if self.path: content = Path(self.path).read_bytes() elif self.url: - response = httpx.get(self.url) + with httpx.Client(follow_redirects=True, max_redirects=3) as client: + response = client.get(self.url) response.raise_for_status() content = response.content return content diff --git a/tests/test_attachments.py b/tests/test_attachments.py index d523e4745..751ec8019 100644 --- a/tests/test_attachments.py +++ b/tests/test_attachments.py @@ -2,6 +2,7 @@ import sys from unittest.mock import ANY +import httpx import pytest from click.testing import CliRunner @@ -97,3 +98,64 @@ def test_attachment_no_file_descriptor_leak(tmp_path): # File descriptor count should not have grown significantly assert _count_open_fds() <= baseline + 5 + + +def test_attachment_content_bytes_follows_redirects(httpx_mock): + httpx_mock.add_response( + url="https://example.com/redirected.png", + status_code=301, + headers={"Location": "https://example.com/actual.png"}, + ) + httpx_mock.add_response( + url="https://example.com/actual.png", + content=TINY_PNG, + ) + attachment = llm.Attachment(url="https://example.com/redirected.png") + assert attachment.content_bytes() == TINY_PNG + + +def test_attachment_content_bytes_limits_redirects(httpx_mock): + for redirect in range(4): + httpx_mock.add_response( + url=f"https://example.com/redirect-{redirect}", + status_code=301, + headers={"Location": f"https://example.com/redirect-{redirect + 1}"}, + ) + + attachment = llm.Attachment(url="https://example.com/redirect-0") + with pytest.raises(httpx.TooManyRedirects): + attachment.content_bytes() + + assert len(httpx_mock.get_requests()) == 4 + + +def test_attachment_resolve_type_follows_redirects(httpx_mock): + httpx_mock.add_response( + method="HEAD", + url="https://example.com/redirected.png", + status_code=301, + headers={"Location": "https://example.com/actual.png"}, + ) + httpx_mock.add_response( + method="HEAD", + url="https://example.com/actual.png", + headers={"content-type": "image/png"}, + ) + attachment = llm.Attachment(url="https://example.com/redirected.png") + assert attachment.resolve_type() == "image/png" + + +def test_attachment_resolve_type_limits_redirects(httpx_mock): + for redirect in range(4): + httpx_mock.add_response( + method="HEAD", + url=f"https://example.com/redirect-{redirect}", + status_code=301, + headers={"Location": f"https://example.com/redirect-{redirect + 1}"}, + ) + + attachment = llm.Attachment(url="https://example.com/redirect-0") + with pytest.raises(httpx.TooManyRedirects): + attachment.resolve_type() + + assert len(httpx_mock.get_requests()) == 4 From 3efc0c623adaef3fc2863a9d59e1faaf48326532 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 13:30:18 -0700 Subject: [PATCH 218/258] Project news section of README and docs index https://claude.ai/share/3fa7dce9-9a79-49a7-a5af-fa5811961a78 > Clone simonw/llm from github and look at the news section of the README, then use the simonwillison.net MCP to find all entries tagged with llm and suggest how the README could be updated with more links and maybe with extra context eg dates or putting them in reverse order to make that README section more useful --- README.md | 29 ++++++++++++++++++----------- docs/index.md | 29 ++++++++++++++++++----------- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 9b7dd1eb3..af4592cf1 100644 --- a/README.md +++ b/README.md @@ -142,17 +142,24 @@ Why don't pelicans like to tip waiters? Because they always have a big bill! ``` -More background on this project: - -- [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/) -- [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/) -- [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) -- [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/) -- [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/) -- [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/) -- [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/) - -See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. +## Project news + +- 29th April 2026: [LLM 0.32a0 is a major backwards-compatible refactor](https://simonwillison.net/2026/Apr/29/llm/) +- 11th August 2025: [LLM 0.27, the annotated release notes: GPT-5 and improved tool calling](https://simonwillison.net/2025/Aug/11/llm-027/) +- 27th May 2025: [Large Language Models can run tools in your terminal with LLM 0.26](https://simonwillison.net/2025/May/27/llm-tools/) +- 5th May 2025: [Feed a video to a vision LLM as a sequence of JPEG frames on the CLI (also LLM 0.25)](https://simonwillison.net/2025/May/5/llm-video-frames/) +- 7th April 2025: [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/) +- 28th February 2025: [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/) +- 17th February 2025: [LLM 0.22, the annotated release notes](https://simonwillison.net/2025/Feb/17/llm/) +- 29th October 2024: [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/) +- 26th January 2024: [LLM 0.13: The annotated release notes](https://simonwillison.net/2024/Jan/26/llm/) +- 12th September 2023: [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/) +- 4th September 2023: [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) +- 12th July 2023: [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/) +- 18th May 2023: [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/) +- 4th April 2023: [The original announcement of the llm CLI tool](https://simonwillison.net/2023/Apr/4/llm/) + +For everything else, see [the llm tag](https://simonwillison.net/tags/llm/) on my blog. ## Contents diff --git a/docs/index.md b/docs/index.md index 62abeeb98..d6a691f8e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -102,17 +102,24 @@ Why don't pelicans like to tip waiters? Because they always have a big bill! ``` -More background on this project: - -- [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/) -- [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/) -- [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) -- [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/) -- [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/) -- [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/) -- [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/) - -See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. +## Project news + +- 29th April 2026: [LLM 0.32a0 is a major backwards-compatible refactor](https://simonwillison.net/2026/Apr/29/llm/) +- 11th August 2025: [LLM 0.27, the annotated release notes: GPT-5 and improved tool calling](https://simonwillison.net/2025/Aug/11/llm-027/) +- 27th May 2025: [Large Language Models can run tools in your terminal with LLM 0.26](https://simonwillison.net/2025/May/27/llm-tools/) +- 5th May 2025: [Feed a video to a vision LLM as a sequence of JPEG frames on the CLI (also LLM 0.25)](https://simonwillison.net/2025/May/5/llm-video-frames/) +- 7th April 2025: [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/) +- 28th February 2025: [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/) +- 17th February 2025: [LLM 0.22, the annotated release notes](https://simonwillison.net/2025/Feb/17/llm/) +- 29th October 2024: [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/) +- 26th January 2024: [LLM 0.13: The annotated release notes](https://simonwillison.net/2024/Jan/26/llm/) +- 12th September 2023: [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/) +- 4th September 2023: [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) +- 12th July 2023: [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/) +- 18th May 2023: [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/) +- 4th April 2023: [The original announcement of the llm CLI tool](https://simonwillison.net/2023/Apr/4/llm/) + +For everything else, see [the llm tag](https://simonwillison.net/tags/llm/) on my blog. ## Contents From 796b167c8140ea398ea0a915288d5c7b375422b4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 13:32:58 -0700 Subject: [PATCH 219/258] Fixed Anthropic example in intro --- README.md | 2 +- docs/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index af4592cf1..7392c8a2a 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ llm -m gemini-3.5-flash 'Tell me fun facts about Mountain View' llm install llm-anthropic llm keys set anthropic # Paste Anthropic API key here -llm -m claude-5-opus 'Impress me with wild facts about turnips' +llm -m claude-sonnet-5 'Impress me with wild facts about turnips' ``` You can also [install a plugin](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#installing-plugins) to access models that can run on your local device. If you use [Ollama](https://ollama.com/): diff --git a/docs/index.md b/docs/index.md index d6a691f8e..b144b9319 100644 --- a/docs/index.md +++ b/docs/index.md @@ -75,7 +75,7 @@ llm -m gemini-3.5-flash 'Tell me fun facts about Mountain View' llm install llm-anthropic llm keys set anthropic # Paste Anthropic API key here -llm -m claude-5-opus 'Impress me with wild facts about turnips' +llm -m claude-sonnet-5 'Impress me with wild facts about turnips' ``` You can also {ref}`install a plugin ` to access models that can run on your local device. If you use [Ollama](https://ollama.com/): ```bash From e8ce4c4299984a28f0119a7ccab54259cdc4902e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 13:38:22 -0700 Subject: [PATCH 220/258] Updated changelog, refs #1588, #1579, #1585 --- docs/changelog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 8bb1687fb..18177eba8 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -4,9 +4,12 @@ ## Unreleased - `llm tools` now provides improved output for dynamic toolboxes - toolbox classes such as `MCP` from [llm-mcp-client](https://github.com/simonw/llm-mcp-client) that generate their tools at runtime. These were previously listed as just their name. They are now listed with their constructor signature and class docstring. Passing one or more specifications such as `llm tools 'MCP("https://datasette.simonwillison.net/-/mcp")'` instantiates each toolbox and lists the tools that configured instance provides. `llm tools --json` output now includes a `"dynamic"` boolean key for each toolbox. [#1580](https://github.com/simonw/llm/issues/1580) +- OpenAI models now support a `service_tier` option. Use `-o service_tier fast` for faster responses at a higher price, or `-o service_tier flex` for slower, cheaper processing on supported models. See {ref}`Fast mode and service tiers `. [#1585](https://github.com/simonw/llm/pull/1585) - Raw provider response payloads are persisted again: the full `response.json()` dictionary is stored in a new `turns.response_json` column, {ref}`condensed ` using [condense-json](https://github.com/simonw/condense-json) 1.1. `llm logs --json` shows the payload resolved back to what the provider sent, and `LogStore.turn_response_json(turn_id)` returns it from Python. This restores data that only lives in the raw payload, such as the log probabilities returned by OpenAI models with `-o logprobs`. [#1586](https://github.com/simonw/llm/pull/1586) - Model plugins can declare a {ref}`json_replacements ` class attribute to further reduce the size of the condendsed JSON. The OpenAI plugin declares dictionaries for both its API shapes - on a sample of real logged turns these took total payload storage from 22% saved with string matching alone to 51%. - Fixed streamed OpenAI Responses API calls recording two different ciphertexts of the same reasoning: the part's `encrypted_content` was harvested from the `response.output_item.done` event while `response_json` came from `response.completed`, and OpenAI encrypts per event. Reasoning metadata is now re-emitted from the final payload, so both records agree on one blob. +- Attachments loaded from URLs now follow up to three redirects when detecting their content type or fetching their bytes. Thanks, [Ojas Sharma](https://github.com/ojassharma7). [#1046](https://github.com/simonw/llm/issues/1046), [#1579](https://github.com/simonw/llm/pull/1579) +- Fixed a bug where `llm openai endpoint --schema` was ignored if the selected template also defined a schema. Thanks, [ikatyal2110](https://github.com/ikatyal2110). [#1588](https://github.com/simonw/llm/pull/1588) (v0_32_rc2)= ## 0.32rc2 (2026-07-30) From 5e2dc959e07c636a024b6900e5618cb492fcd931 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 14:02:19 -0700 Subject: [PATCH 221/258] Fix condensed typo Refs #1590 --- docs/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 18177eba8..28e3ee98f 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -6,7 +6,7 @@ - `llm tools` now provides improved output for dynamic toolboxes - toolbox classes such as `MCP` from [llm-mcp-client](https://github.com/simonw/llm-mcp-client) that generate their tools at runtime. These were previously listed as just their name. They are now listed with their constructor signature and class docstring. Passing one or more specifications such as `llm tools 'MCP("https://datasette.simonwillison.net/-/mcp")'` instantiates each toolbox and lists the tools that configured instance provides. `llm tools --json` output now includes a `"dynamic"` boolean key for each toolbox. [#1580](https://github.com/simonw/llm/issues/1580) - OpenAI models now support a `service_tier` option. Use `-o service_tier fast` for faster responses at a higher price, or `-o service_tier flex` for slower, cheaper processing on supported models. See {ref}`Fast mode and service tiers `. [#1585](https://github.com/simonw/llm/pull/1585) - Raw provider response payloads are persisted again: the full `response.json()` dictionary is stored in a new `turns.response_json` column, {ref}`condensed ` using [condense-json](https://github.com/simonw/condense-json) 1.1. `llm logs --json` shows the payload resolved back to what the provider sent, and `LogStore.turn_response_json(turn_id)` returns it from Python. This restores data that only lives in the raw payload, such as the log probabilities returned by OpenAI models with `-o logprobs`. [#1586](https://github.com/simonw/llm/pull/1586) -- Model plugins can declare a {ref}`json_replacements ` class attribute to further reduce the size of the condendsed JSON. The OpenAI plugin declares dictionaries for both its API shapes - on a sample of real logged turns these took total payload storage from 22% saved with string matching alone to 51%. +- Model plugins can declare a {ref}`json_replacements ` class attribute to further reduce the size of the condensed JSON. The OpenAI plugin declares dictionaries for both its API shapes - on a sample of real logged turns these took total payload storage from 22% saved with string matching alone to 51%. - Fixed streamed OpenAI Responses API calls recording two different ciphertexts of the same reasoning: the part's `encrypted_content` was harvested from the `response.output_item.done` event while `response_json` came from `response.completed`, and OpenAI encrypts per event. Reasoning metadata is now re-emitted from the final payload, so both records agree on one blob. - Attachments loaded from URLs now follow up to three redirects when detecting their content type or fetching their bytes. Thanks, [Ojas Sharma](https://github.com/ojassharma7). [#1046](https://github.com/simonw/llm/issues/1046), [#1579](https://github.com/simonw/llm/pull/1579) - Fixed a bug where `llm openai endpoint --schema` was ignored if the selected template also defined a schema. Thanks, [ikatyal2110](https://github.com/ikatyal2110). [#1588](https://github.com/simonw/llm/pull/1588) From 6b59d561d856ecfec7b8770e24f3618730379f72 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 15:12:02 -0700 Subject: [PATCH 222/258] Upgrade llm logs status to handle turns as well Refs https://github.com/simonw/llm/issues/1590#issuecomment-5171662793 --- docs/logging.md | 4 ++-- llm/cli.py | 9 +++++++-- tests/test_llm_logs.py | 17 +++++++++++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/docs/logging.md b/docs/logging.md index ac4c16088..ad0826b95 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -41,8 +41,8 @@ Example output: ``` Logging is ON for all prompts Found log database at /Users/simon/Library/Application Support/io.datasette.llm/logs.db -Number of conversations logged: 33 -Number of responses logged: 48 +Number of threads logged: 33 +Number of turns logged: 48 Database file size: 19.96MB ``` diff --git a/llm/cli.py b/llm/cli.py index d8163efd5..f05b990df 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -1580,8 +1580,13 @@ def logs_status(): db = sqlite_utils.Database(path) migrate(db) click.echo(f"Found log database at {path}") - click.echo("Number of conversations logged:\t{}".format(db["conversations"].count)) - click.echo("Number of responses logged:\t{}".format(db["responses"].count)) + click.echo("Number of threads logged:\t{}".format(db["threads"].count)) + click.echo("Number of turns logged:\t\t{}".format(db["turns"].count)) + legacy_conversations = db["conversations"].count + legacy_responses = db["responses"].count + if legacy_conversations or legacy_responses: + click.echo(f"Number of legacy conversations:\t{legacy_conversations}") + click.echo(f"Number of legacy responses:\t{legacy_responses}") click.echo(f"Database file size: \t\t{_human_readable_size(path.stat().st_size)}") diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index 51446f123..28eec3b9e 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -1198,6 +1198,23 @@ def test_logs_backup(logs_db): assert expected_path.exists() +def test_logs_status_counts_threads_and_turns(logs_db): + runner = CliRunner() + runner.invoke(cli, ["-m", "echo", "simple prompt"]) + result = runner.invoke(cli, ["logs", "status"]) + assert result.exit_code == 0 + assert "Number of threads logged:\t1" in result.output + assert "Number of turns logged:\t\t1" in result.output + # No legacy rows, so the legacy lines should be hidden + assert "legacy" not in result.output + # Legacy counts show up once legacy tables have rows + logs_db["conversations"].insert({"id": "abc", "name": "test", "model": "echo"}) + result2 = runner.invoke(cli, ["logs", "status"]) + assert result2.exit_code == 0 + assert "Number of legacy conversations:\t1" in result2.output + assert "Number of legacy responses:\t0" in result2.output + + @pytest.mark.parametrize("async_", (False, True)) def test_logs_resolved_model(logs_db, mock_model, async_mock_model, async_): mock_model.resolved_model_name = "resolved-mock" From 3b05ca9df50fc4bbbaffe56246eb574387890882 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 15:17:45 -0700 Subject: [PATCH 223/258] Response.to_dict() forces unconsumed response Refs https://github.com/simonw/llm/issues/1590#issuecomment-5171662793 --- llm/models.py | 1 + tests/test_serialization.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/llm/models.py b/llm/models.py index 0a8098991..9ceef4ab4 100644 --- a/llm/models.py +++ b/llm/models.py @@ -1667,6 +1667,7 @@ def to_dict(self) -> ResponseDict: Returns :class:`~llm.serialization.ResponseDict`. """ + self._force() return _response_to_dict(self) @classmethod diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 2de04c444..d4c86b3e7 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -191,6 +191,16 @@ def test_mock_response_to_dict_matches(self, mock_model): d = r.to_dict() TypeAdapter(ResponseDict).validate_python(d) + def test_to_dict_forces_unconsumed_response(self, mock_model): + mock_model.enqueue(["answer"]) + r = mock_model.prompt("q") + # No text()/messages access first - to_dict() must execute the prompt + d = r.to_dict() + TypeAdapter(ResponseDict).validate_python(d) + assert d["messages"] == [ + {"role": "assistant", "parts": [{"type": "text", "text": "answer"}]} + ] + def test_response_with_reasoning_matches(self, mock_model): mock_model.enqueue( [ From 9e29023b9d613079bce04c90548942ddc933b741 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 16:15:17 -0700 Subject: [PATCH 224/258] Rebuild response._tool_calls from parts Refs https://github.com/simonw/llm/issues/1590#issuecomment-5171662793 --- llm/models.py | 38 ++++++++++++++++++++++----- tests/test_serialization.py | 51 +++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 6 deletions(-) diff --git a/llm/models.py b/llm/models.py index 9ceef4ab4..cbeafc7db 100644 --- a/llm/models.py +++ b/llm/models.py @@ -1585,6 +1585,22 @@ def _response_from_dict( # full picture (reasoning, tool calls, signatures) without needing # a StreamEvent replay. response._loaded_messages = output_messages + # Rebuild _tool_calls from the restored parts so tool_calls() and + # reply(tools=...) can execute serialized pending calls. Server- + # executed calls stay out, matching add_tool_call() during live + # streaming. + from .parts import ToolCallPart + + response._tool_calls = [ + ToolCall( + name=p.name, + arguments=p.arguments or {}, + tool_call_id=p.tool_call_id, + ) + for m in output_messages + for p in m.parts + if isinstance(p, ToolCallPart) and not p.server_executed + ] response._done = True # Restore usage if present. usage = data.get("usage") @@ -1626,12 +1642,12 @@ def reply( from .parts import Message, TextPart, ToolResultPart self._force() - if tool_results is None and self._tool_calls: - tool_results = self.execute_tool_calls() # Forward original tools so the next turn can call them again # (mirrors Conversation.prompt's `tools or self.tools` rule). if "tools" not in kwargs and self.prompt.tools: kwargs["tools"] = self.prompt.tools + if tool_results is None and self._tool_calls: + tool_results = self.execute_tool_calls(tools=kwargs.get("tools")) chain: list[Any] = list(self.prompt.messages) + list(self._messages_now()) if tool_results: chain.append( @@ -1720,15 +1736,20 @@ def execute_tool_calls( before_call: BeforeCallSync | None = None, after_call: AfterCallSync | None = None, tool_calls_list: list[ToolCall] | None = None, + tools: list[ToolDef] | None = None, ) -> list[ToolResult]: """Execute tool calls using this response's tools. By default executes ``self.tool_calls()``; pass ``tool_calls_list=`` to execute an explicit list instead (used when resuming a chain whose history ends in unresolved calls). + Pass ``tools=`` to resolve implementations from an explicit + list instead of ``self.prompt.tools`` (used when a rehydrated + response has pending calls but no tool implementations). """ tool_results = [] - tools_by_name = {tool.name: tool for tool in self.prompt.tools} + effective_tools = _wrap_tools(tools) if tools is not None else self.prompt.tools + tools_by_name = {tool.name: tool for tool in effective_tools} if tool_calls_list is None: tool_calls_list = self.tool_calls() @@ -1992,10 +2013,10 @@ async def reply( raise ValueError( "Response not yet awaited — call `await response` before reply()" ) - if tool_results is None and self._tool_calls: - tool_results = await self.execute_tool_calls() if "tools" not in kwargs and self.prompt.tools: kwargs["tools"] = self.prompt.tools + if tool_results is None and self._tool_calls: + tool_results = await self.execute_tool_calls(tools=kwargs.get("tools")) chain: list[Any] = list(self.prompt.messages) + list(self._messages_now()) if tool_results: chain.append( @@ -2071,16 +2092,21 @@ async def execute_tool_calls( before_call: BeforeCallAsync | None = None, after_call: AfterCallAsync | None = None, tool_calls_list: list[ToolCall] | None = None, + tools: list[ToolDef] | None = None, ) -> list[ToolResult]: """Execute tool calls using this response's tools. By default executes ``await self.tool_calls()``; pass ``tool_calls_list=`` to execute an explicit list instead (used when resuming a chain whose history ends in unresolved calls). + Pass ``tools=`` to resolve implementations from an explicit + list instead of ``self.prompt.tools`` (used when a rehydrated + response has pending calls but no tool implementations). """ if tool_calls_list is None: tool_calls_list = await self.tool_calls() - tools_by_name = {tool.name: tool for tool in self.prompt.tools} + effective_tools = _wrap_tools(tools) if tools is not None else self.prompt.tools + tools_by_name = {tool.name: tool for tool in effective_tools} # Run async prepare_async() on all Toolbox instances that need it instances_to_prepare: list[Toolbox] = [] diff --git a/tests/test_serialization.py b/tests/test_serialization.py index d4c86b3e7..afc5a4e6a 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -201,6 +201,57 @@ def test_to_dict_forces_unconsumed_response(self, mock_model): {"role": "assistant", "parts": [{"type": "text", "text": "answer"}]} ] + def test_from_dict_restores_pending_tool_calls(self, mock_model): + mock_model.enqueue( + [ + llm.parts.StreamEvent( + type="tool_call_name", + chunk="search", + part_index=0, + tool_call_id="c1", + ), + llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"q": "weather"}', + part_index=0, + tool_call_id="c1", + ), + ] + ) + r = mock_model.prompt("find weather") + r2 = llm.Response.from_dict(r.to_dict(), model=mock_model) + assert r2.tool_calls() == [ + llm.ToolCall(name="search", arguments={"q": "weather"}, tool_call_id="c1") + ] + + def search(q: str) -> str: + return f"results for {q}" + + mock_model.enqueue(["it is sunny"]) + r3 = r2.reply(tools=[search]) + r3.text() + tool_message = next(m for m in r3.prompt.messages if m.role == "tool") + result_part = tool_message.parts[0] + assert result_part.output == "results for weather" + assert result_part.exception is None + + def test_from_dict_excludes_server_executed_tool_calls(self, mock_model): + mock_model.enqueue( + [ + llm.parts.StreamEvent( + type="tool_call_name", + chunk="web_search", + part_index=0, + tool_call_id="s1", + server_executed=True, + ), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) + r = mock_model.prompt("q") + r2 = llm.Response.from_dict(r.to_dict(), model=mock_model) + assert r2.tool_calls() == [] + def test_response_with_reasoning_matches(self, mock_model): mock_model.enqueue( [ From 6afc1c494bfa2ef1d9d3b304b0496432a1a8f0fa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 16:28:27 -0700 Subject: [PATCH 225/258] Fix for response.reply() with tools returning attachments Refs https://github.com/simonw/llm/issues/1590#issuecomment-5172813991 --- llm/models.py | 42 ++++++------------------ tests/test_parts.py | 78 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 32 deletions(-) diff --git a/llm/models.py b/llm/models.py index cbeafc7db..ff3a8cf54 100644 --- a/llm/models.py +++ b/llm/models.py @@ -1639,7 +1639,7 @@ def reply( explicit ``tool_results=`` list (e.g. results you mutated, or synthetic ones for testing) to skip auto-execution. """ - from .parts import Message, TextPart, ToolResultPart + from .parts import Message, TextPart self._force() # Forward original tools so the next turn can call them again @@ -1650,21 +1650,10 @@ def reply( tool_results = self.execute_tool_calls(tools=kwargs.get("tools")) chain: list[Any] = list(self.prompt.messages) + list(self._messages_now()) if tool_results: - chain.append( - Message( - role="tool", - parts=[ - ToolResultPart( - name=tr.name, - output=tr.output, - tool_call_id=tr.tool_call_id, - exception=_format_tool_exception(tr.exception), - attachments=list(tr.attachments or []), - ) - for tr in tool_results - ], - ) - ) + tool_attachments: list[Attachment] = [] + for tr in tool_results: + tool_attachments.extend(tr.attachments or []) + _append_tool_results_to_chain(chain, tool_results, tool_attachments) if prompt: chain.append(Message(role="user", parts=[TextPart(text=prompt)])) if messages: @@ -2007,7 +1996,7 @@ async def reply( self.execute_tool_calls()``. See ``Response.reply`` for the ``tool_results=`` semantics. """ - from .parts import Message, TextPart, ToolResultPart + from .parts import Message, TextPart if not self._done: raise ValueError( @@ -2019,21 +2008,10 @@ async def reply( tool_results = await self.execute_tool_calls(tools=kwargs.get("tools")) chain: list[Any] = list(self.prompt.messages) + list(self._messages_now()) if tool_results: - chain.append( - Message( - role="tool", - parts=[ - ToolResultPart( - name=tr.name, - output=tr.output, - tool_call_id=tr.tool_call_id, - exception=_format_tool_exception(tr.exception), - attachments=list(tr.attachments or []), - ) - for tr in tool_results - ], - ) - ) + tool_attachments: list[Attachment] = [] + for tr in tool_results: + tool_attachments.extend(tr.attachments or []) + _append_tool_results_to_chain(chain, tool_results, tool_attachments) if prompt: chain.append(Message(role="user", parts=[TextPart(text=prompt)])) if messages: diff --git a/tests/test_parts.py b/tests/test_parts.py index d3da32876..c21bdb402 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -1468,6 +1468,84 @@ def test_reply_from_conversation_response_extends_chain(self, mock_model): llm.user("q2"), ] + def test_reply_tool_result_attachments_become_user_parts(self, mock_model): + """Attachments returned by tools must surface as a user-role + AttachmentPart message - adapters only emit ToolResultPart.output, + so attachments nested inside the part would be silently lost.""" + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + r1 = mock_model.prompt("q1") + r1.text() + attachment = llm.Attachment(type="image/png", content=b"fakepng") + r2 = r1.reply( + "describe it", + tool_results=[ + llm.ToolResult( + name="take_photo", + output="took photo", + attachments=[attachment], + tool_call_id="c1", + ) + ], + ) + r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.Message( + role="tool", + parts=[ + llm.parts.ToolResultPart( + name="take_photo", output="took photo", tool_call_id="c1" + ) + ], + ), + llm.Message( + role="user", + parts=[llm.parts.AttachmentPart(attachment=attachment)], + ), + llm.user("describe it"), + ] + + @pytest.mark.asyncio + async def test_async_reply_tool_result_attachments_become_user_parts( + self, async_mock_model + ): + async_mock_model.enqueue(["a1"]) + async_mock_model.enqueue(["a2"]) + r1 = async_mock_model.prompt("q1") + await r1.text() + attachment = llm.Attachment(type="image/png", content=b"fakepng") + r2 = await r1.reply( + "describe it", + tool_results=[ + llm.ToolResult( + name="take_photo", + output="took photo", + attachments=[attachment], + tool_call_id="c1", + ) + ], + ) + await r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.Message( + role="tool", + parts=[ + llm.parts.ToolResultPart( + name="take_photo", output="took photo", tool_call_id="c1" + ) + ], + ), + llm.Message( + role="user", + parts=[llm.parts.AttachmentPart(attachment=attachment)], + ), + llm.user("describe it"), + ] + @pytest.mark.asyncio async def test_async_reply(self, async_mock_model): async_mock_model.enqueue(["a1"]) From 7a9454f87637b871af18be84361eb6a94eb0a9ec Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 17:22:28 -0700 Subject: [PATCH 226/258] Draft release notes for 0.32, refs #1590 --- docs/changelog.md | 63 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 28e3ee98f..30a2439a6 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,15 +1,68 @@ # Changelog -(unreleased)= -## Unreleased +(v0_32)= +## 0.32 (2026-08-03) -- `llm tools` now provides improved output for dynamic toolboxes - toolbox classes such as `MCP` from [llm-mcp-client](https://github.com/simonw/llm-mcp-client) that generate their tools at runtime. These were previously listed as just their name. They are now listed with their constructor signature and class docstring. Passing one or more specifications such as `llm tools 'MCP("https://datasette.simonwillison.net/-/mcp")'` instantiates each toolbox and lists the tools that configured instance provides. `llm tools --json` output now includes a `"dynamic"` boolean key for each toolbox. [#1580](https://github.com/simonw/llm/issues/1580) +LLM 0.32 is a major, backwards-compatible update to the way prompts, responses, tools and logs are represented. It adds structured messages and parts throughout the Python API, adopts the OpenAI Responses API for reasoning-capable models, substantially expands control over pausable and resumable tool loops and introduces a new content-addressed SQLite logging schema. + +### Structured messages and richer responses + +Prompt inputs and model outputs are now represented as lists of `Message` objects, each containing typed `Part` objects for text, reasoning, tool calls, tool results and attachments. + +- New `llm.Message` value type and constructor helpers `llm.user()`, `llm.assistant()`, `llm.system()` and `llm.tool_message()`. +- New `messages=` keyword argument on the prompt, conversation and chain APIs, including their asynchronous equivalents. Existing `prompt=`, `system=`, `attachments=` and `tool_results=` arguments continue to work and are converted into the same structured representation. +- New `response.stream_events()` and `response.astream_events()` methods expose mixed streams of text, reasoning, tool calls and tool results. Iterating over a response directly continues to yield text strings. +- `response.messages()` returns the assembled structured output. `response.reply()` continues from any response and can automatically execute pending tool calls before the next turn. +- `response.to_dict()` and `Response.from_dict()` provide JSON-safe persistence of complete turns, including reasoning and provider metadata, with matching `TypedDict` definitions in the new `llm.serialization` module. +- `response.prompt.messages` is now the canonical record of exactly what was sent to the model, including the full preceding conversation chain. + +These APIs were introduced in {ref}`0.32a0 `, which has a more detailed inventory and links to the expanded {ref}`Advanced model plugins ` documentation. + +### OpenAI Responses API and reasoning + +- Most reasoning-capable OpenAI models now use the `/v1/responses` endpoint by default, enabling interleaved reasoning across tool calls. The existing Chat Completions classes remain available, and `-o chat_completions 1` selects that older path for an individual prompt. See {ref}`0.32a2 ` for the full list of affected models. +- Visible reasoning summaries are streamed to standard error by `llm prompt` and `llm chat`. Use `-R/--hide-reasoning` or the new `hide_reasoning=True` Python argument to hide them. Encrypted reasoning metadata is preserved for subsequent turns. +- The default model for users who have not selected one is now [GPT-5.6 Luna](https://developers.openai.com/api/docs/models/gpt-5.6-luna), replacing GPT-4o mini. New built-in models include `gpt-5.6-sol`, `gpt-5.6-terra` and `gpt-5.6-luna`; models that are no longer available from OpenAI have been removed. See {ref}`0.32rc1 ` and {ref}`0.32rc2 ` for details. +- New {ref}`llm openai endpoint ` command runs prompts and chats, or lists models, against an arbitrary OpenAI-compatible endpoint without configuring it first. These calls are not logged. - OpenAI models now support a `service_tier` option. Use `-o service_tier fast` for faster responses at a higher price, or `-o service_tier flex` for slower, cheaper processing on supported models. See {ref}`Fast mode and service tiers `. [#1585](https://github.com/simonw/llm/pull/1585) -- Raw provider response payloads are persisted again: the full `response.json()` dictionary is stored in a new `turns.response_json` column, {ref}`condensed ` using [condense-json](https://github.com/simonw/condense-json) 1.1. `llm logs --json` shows the payload resolved back to what the provider sent, and `LogStore.turn_response_json(turn_id)` returns it from Python. This restores data that only lives in the raw payload, such as the log probabilities returned by OpenAI models with `-o logprobs`. [#1586](https://github.com/simonw/llm/pull/1586) -- Model plugins can declare a {ref}`json_replacements ` class attribute to further reduce the size of the condensed JSON. The OpenAI plugin declares dictionaries for both its API shapes - on a sample of real logged turns these took total payload storage from 22% saved with string matching alone to 51%. +- New `llm -m model --options` flag lists the options supported by a model. The Python prompt APIs now accept an explicit `options=` dictionary as well as the previous keyword-argument form. + +### More controllable tool loops + +- Every tool call now has a unique `tool_call_id`, synthesized when the provider does not supply one. Tool implementations can accept an `llm_tool_call` parameter to inspect the current call and its ID. +- Tools can raise `llm.PauseChain` to pause execution for human approval or another external event. Chains can later resume from a message history ending in unresolved tool calls, without repeating calls that already have results. +- `response.execute_tool_call()` executes one call, while `response.execute_tool_calls(tool_calls_list=...)` can execute an explicit list. Both are awaitable on asynchronous responses. +- Async sibling calls finish before a pause or callback failure is propagated, avoiding orphaned work. Missing async tools now produce the same error results as synchronous execution. +- Conversations that use configured toolboxes can be continued with `llm -c` or `llm chat -c` without repeating the toolbox configuration. +- `llm tools` now shows constructor signatures and docstrings for dynamic toolboxes. Passing a toolbox specification instantiates it and lists its runtime-generated tools, while `llm tools --json` identifies dynamic toolboxes with a `"dynamic"` boolean. [#1580](https://github.com/simonw/llm/issues/1580) + +See {ref}`0.32a3 ` for more detail on pausing, resuming and inspecting tool calls. + +### New SQLite logging schema + +LLM now logs prompts and responses using a new schema built around threads, turns and a **content-addressed message store**. Existing records in the legacy `responses` table are left untouched, and `llm logs` combines both generations of data. You can create a backup before upgrading using: + + llm logs backup logs-backup.db + +- Messages are stored once and referenced by their content hash, preserving structured text, reasoning, attachments and tool activity without duplicating repeated conversation history. See {ref}`the message store documentation `. +- Raw provider payloads are stored in `turns.response_json`, {ref}`condensed ` using [condense-json](https://github.com/simonw/condense-json). `llm logs --json` expands them back to their original shape, and `LogStore.turn_response_json(turn_id)` returns them from Python. [#1586](https://github.com/simonw/llm/pull/1586) +- Model plugins can define {ref}`json_replacements ` dictionaries to further improve payload compression. +- Full-text search, model and tool filters and conversation continuation work across both the legacy and new tables. Logs now record which configured toolbox instance supplied each tool. +- New {ref}`Response.log_to_db() ` Python API writes a response to a logs database. `llm prompt --json` outputs the same structured representation as `llm logs --json`, even when persistent logging is disabled. +- New {ref}`message_tree SQL view ` renders conversation threads as indented text outlines for direct SQL exploration. +- LLM now requires [sqlite-utils 4.0](https://sqlite-utils.datasette.io/en/stable/changelog.html#v4-0) or higher and no longer depends on `sqlite-migrate`. + +See {ref}`0.32rc1 ` for the detailed migration notes and complete list of logging changes. + +### Fixes since 0.32rc2 + - Fixed streamed OpenAI Responses API calls recording two different ciphertexts of the same reasoning: the part's `encrypted_content` was harvested from the `response.output_item.done` event while `response_json` came from `response.completed`, and OpenAI encrypts per event. Reasoning metadata is now re-emitted from the final payload, so both records agree on one blob. - Attachments loaded from URLs now follow up to three redirects when detecting their content type or fetching their bytes. Thanks, [Ojas Sharma](https://github.com/ojassharma7). [#1046](https://github.com/simonw/llm/issues/1046), [#1579](https://github.com/simonw/llm/pull/1579) - Fixed a bug where `llm openai endpoint --schema` was ignored if the selected template also defined a schema. Thanks, [ikatyal2110](https://github.com/ikatyal2110). [#1588](https://github.com/simonw/llm/pull/1588) +- `llm logs status` now counts records in the new `threads` and `turns` tables, with legacy conversation and response counts shown separately when present. [#1590](https://github.com/simonw/llm/issues/1590) +- `Response.to_dict()` now executes an unconsumed synchronous response before serializing it instead of producing an empty assistant message list. [#1590](https://github.com/simonw/llm/issues/1590) +- `Response.from_dict()` now restores pending client-side tool calls so they can be inspected, executed or continued using `response.reply(tools=[...])`. [#1590](https://github.com/simonw/llm/issues/1590) +- `Response.reply()` now correctly passes attachments returned by tools to the next model call, for both synchronous and asynchronous responses. [#1590](https://github.com/simonw/llm/issues/1590) (v0_32_rc2)= ## 0.32rc2 (2026-07-30) From db7b4eed21716cfde99a0d7df568d9c5b373169a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 17:36:34 -0700 Subject: [PATCH 227/258] Assemble multiple assistant messages via StreamEvent.message_index Refs https://github.com/simonw/llm/issues/1590#issuecomment-5172892540, #1591 --- llm/default_plugins/openai_models.py | 297 ++++++++++++++++++++------- llm/models.py | 140 +++++++++---- llm/parts.py | 7 +- tests/test_openai_responses.py | 108 ++++++++++ tests/test_parts.py | 67 ++++++ 5 files changed, 503 insertions(+), 116 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 2cb55b97e..634969320 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1740,6 +1740,12 @@ def _build_responses_input(self, prompt, image_detail=None): ) ) elif isinstance(part, ToolCallPart): + if part.server_executed: + # Server-side tool calls (web_search, + # code_interpreter) ran inside OpenAI's + # infrastructure - they must not be replayed as + # client function_call items. + continue tool_call_items.append( { "type": "function_call", @@ -1749,6 +1755,8 @@ def _build_responses_input(self, prompt, image_detail=None): } ) elif isinstance(part, ToolResultPart): + if part.server_executed: + continue tool_result_items.append( { "type": "function_call_output", @@ -1963,10 +1971,163 @@ def _reasoning_refresh_events(self, response_json, done_events): chunk="", part_index=prior.part_index, provider_metadata={"openai": meta}, + message_index=prior.message_index, ) ) return events + def _server_tool_events(self, item, message_index): + """StreamEvents for a server-side tool call output item + (web_search_call / code_interpreter_call), or [] for other + item types. The call and its result both carry + ``server_executed=True`` so they are recorded in the message + parts without entering the locally-executable tool call list. + """ + item_type = getattr(item, "type", None) + item_id = getattr(item, "id", None) + events: list[StreamEvent] = [] + if item_type == "web_search_call": + action = getattr(item, "action", None) + if hasattr(action, "model_dump"): + action = action.model_dump() + events.append( + StreamEvent( + type="tool_call_name", + chunk="web_search", + tool_call_id=item_id, + server_executed=True, + message_index=message_index, + ) + ) + events.append( + StreamEvent( + type="tool_call_args", + chunk=json.dumps(action or {}), + tool_call_id=item_id, + server_executed=True, + message_index=message_index, + ) + ) + # Search results are not included in the call item - they + # surface as citations in the following message - so the + # result records completion status only. + events.append( + StreamEvent( + type="tool_result", + chunk=getattr(item, "status", None) or "completed", + tool_call_id=item_id, + server_executed=True, + tool_name="web_search", + message_index=message_index, + ) + ) + elif item_type == "code_interpreter_call": + code = getattr(item, "code", None) or "" + events.append( + StreamEvent( + type="tool_call_name", + chunk="code_interpreter", + tool_call_id=item_id, + server_executed=True, + message_index=message_index, + ) + ) + events.append( + StreamEvent( + type="tool_call_args", + chunk=json.dumps({"code": code}), + tool_call_id=item_id, + server_executed=True, + message_index=message_index, + ) + ) + output_bits = [] + for output in getattr(item, "outputs", None) or []: + if hasattr(output, "model_dump"): + output = output.model_dump() + if isinstance(output, dict): + text = output.get("logs") or output.get("url") + if text: + output_bits.append(text) + events.append( + StreamEvent( + type="tool_result", + chunk="\n".join(output_bits) + or (getattr(item, "status", None) or "completed"), + tool_call_id=item_id, + server_executed=True, + tool_name="code_interpreter", + message_index=message_index, + ) + ) + return events + + def _non_streaming_output_events(self, output, response): + """Translate a non-streaming Responses ``output`` item list into + StreamEvents. Returns ``(events, had_reasoning)``. + + Each ``message`` item after the first starts a new + ``message_index``, so server-side tool execution that + interleaves multiple message items assembles into multiple + assistant Messages. Items between two message items (tool + calls, reasoning) group with the preceding message. + """ + events: list[StreamEvent] = [] + had_reasoning = False + message_index = 0 + seen_message = False + for item in output: + if item.type == "message" and seen_message: + message_index += 1 + if item.type == "reasoning": + had_reasoning = True + event = self._reasoning_event(item) + event.message_index = message_index + events.append(event) + elif item.type == "function_call": + try: + args = json.loads(item.arguments) if item.arguments else {} + except json.JSONDecodeError: + args = {"_raw": item.arguments} + response.add_tool_call( + llm.ToolCall( + tool_call_id=item.call_id, + name=item.name, + arguments=args, + ) + ) + events.append( + StreamEvent( + type="tool_call_name", + chunk=item.name or "", + tool_call_id=item.call_id, + message_index=message_index, + ) + ) + events.append( + StreamEvent( + type="tool_call_args", + chunk=item.arguments or "", + tool_call_id=item.call_id, + message_index=message_index, + ) + ) + elif item.type == "message": + seen_message = True + for content in item.content or []: + ctype = getattr(content, "type", None) + if ctype == "output_text" and content.text: + events.append( + StreamEvent( + type="text", + chunk=content.text, + message_index=message_index, + ) + ) + else: + events.extend(self._server_tool_events(item, message_index)) + return events, had_reasoning + class Responses(_SharedResponses, KeyModel): needs_key = "openai" @@ -2075,11 +2236,17 @@ def execute( final_response_dict: dict[str, Any] | None = None reasoning_items_with_streamed_text = set() reasoning_done_events: dict[str, StreamEvent] = {} + message_index = 0 + seen_message = False for event in stream_obj: etype = getattr(event, "type", None) if etype == "response.output_item.added": item = event.item - if item.type == "function_call": + if item.type == "message": + if seen_message: + message_index += 1 + seen_message = True + elif item.type == "function_call": tool_call_meta[item.id] = { "id": item.id, "call_id": item.call_id, @@ -2089,9 +2256,14 @@ def execute( type="tool_call_name", chunk=item.name or "", tool_call_id=item.call_id, + message_index=message_index, ) elif etype == "response.output_text.delta": - yield StreamEvent(type="text", chunk=event.delta or "") + yield StreamEvent( + type="text", + chunk=event.delta or "", + message_index=message_index, + ) elif etype == "response.function_call_arguments.delta": item_id = getattr(event, "item_id", None) meta = tool_call_meta.get(item_id) if item_id else None @@ -2100,6 +2272,7 @@ def execute( type="tool_call_args", chunk=event.delta or "", tool_call_id=call_id, + message_index=message_index, ) elif etype in ( "response.reasoning_summary_text.delta", @@ -2108,7 +2281,11 @@ def execute( item_id = getattr(event, "item_id", None) if item_id: reasoning_items_with_streamed_text.add(item_id) - yield StreamEvent(type="reasoning", chunk=event.delta or "") + yield StreamEvent( + type="reasoning", + chunk=event.delta or "", + message_index=message_index, + ) elif etype in ( "response.reasoning_summary_text.done", "response.reasoning_text.done", @@ -2119,7 +2296,11 @@ def execute( if text: if item_id: reasoning_items_with_streamed_text.add(item_id) - yield StreamEvent(type="reasoning", chunk=text) + yield StreamEvent( + type="reasoning", + chunk=text, + message_index=message_index, + ) elif etype == "response.output_item.done": item = event.item if item.type == "reasoning": @@ -2131,6 +2312,7 @@ def execute( item_id not in reasoning_items_with_streamed_text ), ) + reasoning_event.message_index = message_index if item_id: # Retained so the refresh after # response.completed can target the part @@ -2149,6 +2331,8 @@ def execute( arguments=args, ) ) + else: + yield from self._server_tool_events(item, message_index) elif etype == "response.completed": final_response_dict = event.response.model_dump() if final_response_dict.get("usage"): @@ -2168,37 +2352,10 @@ def execute( dumped = completion.model_dump() response.response_json = remove_dict_none_values(dumped) usage = dumped.get("usage") - for item in completion.output: - if item.type == "reasoning": - had_reasoning = True - yield self._reasoning_event(item) - elif item.type == "function_call": - try: - args = json.loads(item.arguments) if item.arguments else {} - except json.JSONDecodeError: - args = {"_raw": item.arguments} - response.add_tool_call( - llm.ToolCall( - tool_call_id=item.call_id, - name=item.name, - arguments=args, - ) - ) - yield StreamEvent( - type="tool_call_name", - chunk=item.name or "", - tool_call_id=item.call_id, - ) - yield StreamEvent( - type="tool_call_args", - chunk=item.arguments or "", - tool_call_id=item.call_id, - ) - elif item.type == "message": - for content in item.content or []: - ctype = getattr(content, "type", None) - if ctype == "output_text" and content.text: - yield StreamEvent(type="text", chunk=content.text) + events, had_reasoning = self._non_streaming_output_events( + completion.output, response + ) + yield from events self._set_usage_responses(response, usage) # Fallback: usage said reasoning happened but the API gave us no @@ -2323,11 +2480,17 @@ async def execute( final_response_dict: dict[str, Any] | None = None reasoning_items_with_streamed_text = set() reasoning_done_events: dict[str, StreamEvent] = {} + message_index = 0 + seen_message = False async for event in stream_obj: etype = getattr(event, "type", None) if etype == "response.output_item.added": item = event.item - if item.type == "function_call": + if item.type == "message": + if seen_message: + message_index += 1 + seen_message = True + elif item.type == "function_call": tool_call_meta[item.id] = { "id": item.id, "call_id": item.call_id, @@ -2337,9 +2500,14 @@ async def execute( type="tool_call_name", chunk=item.name or "", tool_call_id=item.call_id, + message_index=message_index, ) elif etype == "response.output_text.delta": - yield StreamEvent(type="text", chunk=event.delta or "") + yield StreamEvent( + type="text", + chunk=event.delta or "", + message_index=message_index, + ) elif etype == "response.function_call_arguments.delta": item_id = getattr(event, "item_id", None) meta = tool_call_meta.get(item_id) if item_id else None @@ -2348,6 +2516,7 @@ async def execute( type="tool_call_args", chunk=event.delta or "", tool_call_id=call_id, + message_index=message_index, ) elif etype in ( "response.reasoning_summary_text.delta", @@ -2356,7 +2525,11 @@ async def execute( item_id = getattr(event, "item_id", None) if item_id: reasoning_items_with_streamed_text.add(item_id) - yield StreamEvent(type="reasoning", chunk=event.delta or "") + yield StreamEvent( + type="reasoning", + chunk=event.delta or "", + message_index=message_index, + ) elif etype in ( "response.reasoning_summary_text.done", "response.reasoning_text.done", @@ -2367,7 +2540,11 @@ async def execute( if text: if item_id: reasoning_items_with_streamed_text.add(item_id) - yield StreamEvent(type="reasoning", chunk=text) + yield StreamEvent( + type="reasoning", + chunk=text, + message_index=message_index, + ) elif etype == "response.output_item.done": item = event.item if item.type == "reasoning": @@ -2379,6 +2556,7 @@ async def execute( item_id not in reasoning_items_with_streamed_text ), ) + reasoning_event.message_index = message_index if item_id: # Retained so the refresh after # response.completed can target the part @@ -2397,6 +2575,11 @@ async def execute( arguments=args, ) ) + else: + for server_event in self._server_tool_events( + item, message_index + ): + yield server_event elif etype == "response.completed": final_response_dict = event.response.model_dump() if final_response_dict.get("usage"): @@ -2417,37 +2600,11 @@ async def execute( dumped = completion.model_dump() response.response_json = remove_dict_none_values(dumped) usage = dumped.get("usage") - for item in completion.output: - if item.type == "reasoning": - had_reasoning = True - yield self._reasoning_event(item) - elif item.type == "function_call": - try: - args = json.loads(item.arguments) if item.arguments else {} - except json.JSONDecodeError: - args = {"_raw": item.arguments} - response.add_tool_call( - llm.ToolCall( - tool_call_id=item.call_id, - name=item.name, - arguments=args, - ) - ) - yield StreamEvent( - type="tool_call_name", - chunk=item.name or "", - tool_call_id=item.call_id, - ) - yield StreamEvent( - type="tool_call_args", - chunk=item.arguments or "", - tool_call_id=item.call_id, - ) - elif item.type == "message": - for content in item.content or []: - ctype = getattr(content, "type", None) - if ctype == "output_text" and content.text: - yield StreamEvent(type="text", chunk=content.text) + events, had_reasoning = self._non_streaming_output_events( + completion.output, response + ) + for event in events: + yield event self._set_usage_responses(response, usage) if ( diff --git a/llm/models.py b/llm/models.py index ff3a8cf54..6fe1126e3 100644 --- a/llm/models.py +++ b/llm/models.py @@ -1049,6 +1049,7 @@ def __init__( self._auto_last_index: int | None = None self._auto_last_family: str | None = None self._auto_tool_id_to_index: dict[str, int] = {} + self._auto_last_message_index: int = 0 self._done = False self._tool_calls: list[ToolCall] = [] self.response_json: dict[str, Any] | None = None @@ -1081,10 +1082,10 @@ def _messages_now(self) -> list[Any]: loaded = getattr(self, "_loaded_messages", None) if loaded is not None: return list(loaded) - parts = self._build_parts() - if not parts: - return [] - return [Message(role="assistant", parts=parts)] + return [ + Message(role="assistant", parts=parts) + for parts in self._build_message_parts() + ] @staticmethod def _event_family(event_type: str) -> str: @@ -1103,6 +1104,13 @@ def _resolve_part_index(self, event): """ fam = self._event_family(event.type) + # A message_index change always starts a fresh part - text on + # either side of a message boundary must not concatenate. + if event.message_index != self._auto_last_message_index: + self._auto_last_family = None + self._auto_last_index = None + self._auto_last_message_index = event.message_index + if event.part_index is not None: self._auto_index_max = max(self._auto_index_max, event.part_index) if ( @@ -1191,7 +1199,20 @@ def _process_chunk(self, chunk): return chunk def _build_parts(self) -> list[Any]: - """Assemble Part objects from the accumulated stream events. + """All Parts from the accumulated stream events, flattened + across message boundaries. See ``_build_message_parts``.""" + return [part for parts in self._build_message_parts() for part in parts] + + def _build_message_parts(self) -> list[list[Any]]: + """Assemble Part objects from the accumulated stream events, + grouped into one parts-list per assistant message. + + Most providers emit a single assistant message, so the result + is usually a one-element list. Events carrying an explicit + ``message_index`` (OpenAI Responses server-side tool execution + interleaves multiple ``message`` output items in one response) + split into one parts-list per distinct index, in first-seen + order. Events sharing a part_index group into one Part. Mixing families (text vs tool_call vs reasoning vs tool_result) at the @@ -1230,23 +1251,26 @@ def _build_parts(self) -> list[Any]: tool_call_id=tc.tool_call_id, ) ) - return fallback_parts + return [fallback_parts] if fallback_parts else [] # Group events by their (resolved) part_index, preserving the # order in which each index was first seen. Then build one Part # per group. This handles non-adjacent same-index events (e.g. # text → tool_call → text where the plugin pinned both text - # bursts to part_index=0) by merging them into one Part. + # bursts to part_index=0) by merging them into one Part. Each + # group belongs to the message of its first event. groups: dict[int, list[Any]] = {} order: list[int] = [] + group_message: dict[int, int] = {} for event in self._stream_events: pi = event.part_index if pi not in groups: groups[pi] = [] order.append(pi) + group_message[pi] = event.message_index groups[pi].append(event) - parts: list[Any] = [] + built: list[tuple[int, Any]] = [] for pi in order: evs = groups[pi] fam_first = self._event_family(evs[0].type) @@ -1266,19 +1290,23 @@ def _build_parts(self) -> list[Any]: merged[k] = v pm_merged = merged + mi = group_message[pi] if fam_first == "text": text = "".join(e.chunk for e in evs) if text: - parts.append(TextPart(text=text, provider_metadata=pm_merged)) + built.append((mi, TextPart(text=text, provider_metadata=pm_merged))) elif fam_first == "reasoning": text = "".join(e.chunk for e in evs) redacted = any(e.redacted for e in evs) if text or redacted: - parts.append( - ReasoningPart( - text=text, - redacted=redacted, - provider_metadata=pm_merged, + built.append( + ( + mi, + ReasoningPart( + text=text, + redacted=redacted, + provider_metadata=pm_merged, + ), ) ) elif fam_first == "tool_call": @@ -1292,13 +1320,16 @@ def _build_parts(self) -> list[Any]: (e.tool_call_id for e in evs if e.tool_call_id), None ) server_executed = any(e.server_executed for e in evs) - parts.append( - ToolCallPart( - name=tool_name, - arguments=arguments, - tool_call_id=tool_call_id, - server_executed=server_executed, - provider_metadata=pm_merged, + built.append( + ( + mi, + ToolCallPart( + name=tool_name, + arguments=arguments, + tool_call_id=tool_call_id, + server_executed=server_executed, + provider_metadata=pm_merged, + ), ) ) elif fam_first == "tool_result": @@ -1307,28 +1338,46 @@ def _build_parts(self) -> list[Any]: (e.tool_call_id for e in evs if e.tool_call_id), None ) server_executed = any(e.server_executed for e in evs) - parts.append( - ToolResultPart( - name=tool_result_name, - output="".join(e.chunk for e in evs), - tool_call_id=tool_call_id, - server_executed=server_executed, - provider_metadata=pm_merged, + built.append( + ( + mi, + ToolResultPart( + name=tool_result_name, + output="".join(e.chunk for e in evs), + tool_call_id=tool_call_id, + server_executed=server_executed, + provider_metadata=pm_merged, + ), ) ) + # Split into per-message parts lists, message indexes in + # first-seen order. + message_order: list[int] = [] + by_message: dict[int, list[Any]] = {} + for mi, part in built: + if mi not in by_message: + by_message[mi] = [] + message_order.append(mi) + by_message[mi].append(part) + messages_parts = [by_message[mi] for mi in message_order] + if not messages_parts: + messages_parts = [[]] + # Merge in any tool calls registered via add_tool_call() that the # plugin didn't also emit as StreamEvents. Dedup by tool_call_id so - # plugins using both APIs in tandem don't double-count. + # plugins using both APIs in tandem don't double-count. They join + # the final message - matching the old append-at-end behavior. seen_ids = { p.tool_call_id + for parts in messages_parts for p in parts if isinstance(p, ToolCallPart) and p.tool_call_id is not None } for tc in self._tool_calls: if tc.tool_call_id is not None and tc.tool_call_id in seen_ids: continue - parts.append( + messages_parts[-1].append( ToolCallPart( name=tc.name, arguments=tc.arguments or {}, @@ -1336,21 +1385,24 @@ def _build_parts(self) -> list[Any]: ) ) - # Hoist redacted reasoning Parts to the start of the assembled - # message. Plugins typically emit them late (when usage arrives - # in the final chunk), but UIs render reasoning before content, - # so the framework reorders. Relative order among redacted - # Parts is preserved. - redacted_parts = [ - p for p in parts if isinstance(p, ReasoningPart) and p.redacted - ] - if redacted_parts: - other_parts = [ - p for p in parts if not (isinstance(p, ReasoningPart) and p.redacted) + # Hoist redacted reasoning Parts to the start of their message. + # Plugins typically emit them late (when usage arrives in the + # final chunk), but UIs render reasoning before content, so the + # framework reorders. Relative order among redacted Parts is + # preserved. + for i, parts in enumerate(messages_parts): + redacted_parts = [ + p for p in parts if isinstance(p, ReasoningPart) and p.redacted ] - parts = redacted_parts + other_parts - - return parts + if redacted_parts: + other_parts = [ + p + for p in parts + if not (isinstance(p, ReasoningPart) and p.redacted) + ] + messages_parts[i] = redacted_parts + other_parts + + return [parts for parts in messages_parts if parts] def add_tool_call(self, tool_call: ToolCall): self._tool_calls.append(_ensure_tool_call_id(tool_call)) diff --git a/llm/parts.py b/llm/parts.py index c7cff8c2b..58f4dcea2 100644 --- a/llm/parts.py +++ b/llm/parts.py @@ -334,8 +334,11 @@ class StreamEvent: it onto the finalized Part (last non-None wins per top-level key). `message_index` is for providers that emit multiple assistant - messages in a single response (Anthropic server-side tool - execution); most plugins leave it at 0. + messages in a single response (OpenAI Responses server-side tool + execution interleaves multiple `message` output items with tool + calls); most plugins leave it at 0 and get a single assistant + Message. Events with distinct indexes assemble into distinct + Messages, in first-seen order. """ type: str # "text" / "reasoning" / "tool_call_name" / diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index 2dfb95ff6..e0a4d1928 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -858,3 +858,111 @@ def test_responses_reasoning_metadata_refreshed_from_final_payload(httpx_mock): payload_item = response.response_json["output"][0] assert payload_item["encrypted_content"] == "encrypted-final" assert reasoning_parts[0].text == "Thinking aloud" + + +def test_code_interpreter_multi_message_response(httpx_mock): + """Server-side tool execution interleaving multiple message output + items must assemble into multiple assistant Messages.""" + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_multi", + "object": "response", + "created_at": 1, + "model": "gpt-5.5", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [ + {"type": "output_text", "text": "STEP ONE", "annotations": []} + ], + }, + { + "type": "code_interpreter_call", + "id": "ci_1", + "status": "completed", + "container_id": "cntr_1", + "code": "print(111*111)", + "outputs": [{"type": "logs", "logs": "12321\n"}], + }, + { + "type": "message", + "id": "msg_2", + "role": "assistant", + "status": "completed", + "content": [ + {"type": "output_text", "text": "DONE 12321", "annotations": []} + ], + }, + ], + "usage": {"input_tokens": 5, "output_tokens": 3, "total_tokens": 8}, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + model = llm.get_model("gpt-5.5") + response = model.prompt("count", stream=False, key="test") + response.text() + + messages = response.messages() + assert len(messages) == 2 + first, second = messages + assert [type(p).__name__ for p in first.parts] == [ + "TextPart", + "ToolCallPart", + "ToolResultPart", + ] + assert first.parts[0].text == "STEP ONE" + assert first.parts[1].name == "code_interpreter" + assert first.parts[1].arguments == {"code": "print(111*111)"} + assert first.parts[1].server_executed + assert first.parts[2].output == "12321\n" + assert first.parts[2].server_executed + assert [type(p).__name__ for p in second.parts] == ["TextPart"] + assert second.parts[0].text == "DONE 12321" + + # Server-executed calls are not locally executable + assert response.tool_calls() == [] + + +def test_server_tool_parts_not_replayed_as_function_calls(): + from llm.parts import Message, TextPart, ToolCallPart, ToolResultPart + + model = llm.get_model("gpt-5.5") + + class FakePrompt: + messages = ( + Message(role="user", parts=[TextPart(text="count")]), + Message( + role="assistant", + parts=[ + TextPart(text="STEP ONE"), + ToolCallPart( + name="code_interpreter", + arguments={"code": "print(1)"}, + tool_call_id="ci_1", + server_executed=True, + ), + ToolResultPart( + name="code_interpreter", + output="1\n", + tool_call_id="ci_1", + server_executed=True, + ), + ], + ), + Message(role="assistant", parts=[TextPart(text="DONE")]), + Message(role="user", parts=[TextPart(text="thanks")]), + ) + + items, _instructions = model._build_responses_input(FakePrompt()) + assert items == [ + {"role": "user", "content": "count"}, + {"role": "assistant", "content": "STEP ONE"}, + {"role": "assistant", "content": "DONE"}, + {"role": "user", "content": "thanks"}, + ] diff --git a/tests/test_parts.py b/tests/test_parts.py index c21bdb402..ffe3b6c08 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -1406,6 +1406,73 @@ def execute(self, prompt, stream, response, conversation): ] +class TestMessageIndexAssembly: + def test_message_index_splits_assistant_messages(self, mock_model): + mock_model.enqueue( + [ + llm.parts.StreamEvent(type="text", chunk="STEP ONE", message_index=0), + llm.parts.StreamEvent( + type="tool_call_name", + chunk="code_interpreter", + tool_call_id="ci1", + server_executed=True, + message_index=0, + ), + llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"code": "print(1)"}', + tool_call_id="ci1", + server_executed=True, + message_index=0, + ), + llm.parts.StreamEvent( + type="tool_result", + chunk="1\n", + tool_call_id="ci1", + server_executed=True, + tool_name="code_interpreter", + message_index=0, + ), + llm.parts.StreamEvent(type="text", chunk="STEP TWO", message_index=1), + ] + ) + response = mock_model.prompt("go") + response.text() + messages = response.messages() + assert len(messages) == 2 + assert all(m.role == "assistant" for m in messages) + assert messages[0].parts == [ + llm.parts.TextPart(text="STEP ONE"), + llm.parts.ToolCallPart( + name="code_interpreter", + arguments={"code": "print(1)"}, + tool_call_id="ci1", + server_executed=True, + ), + llm.parts.ToolResultPart( + name="code_interpreter", + output="1\n", + tool_call_id="ci1", + server_executed=True, + ), + ] + assert messages[1].parts == [llm.parts.TextPart(text="STEP TWO")] + + def test_adjacent_text_across_message_boundary_does_not_merge(self, mock_model): + mock_model.enqueue( + [ + llm.parts.StreamEvent(type="text", chunk="first", message_index=0), + llm.parts.StreamEvent(type="text", chunk="second", message_index=1), + ] + ) + response = mock_model.prompt("go") + response.text() + messages = response.messages() + assert len(messages) == 2 + assert messages[0].parts == [llm.parts.TextPart(text="first")] + assert messages[1].parts == [llm.parts.TextPart(text="second")] + + class TestResponseReply: def test_reply_builds_next_turn_from_this_response(self, mock_model): mock_model.enqueue(["a1"]) From 07f5c402d20269bba38d6f6163f18bbd76785891 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 17:54:21 -0700 Subject: [PATCH 228/258] Implement core server-side tool abstraction --- docs/plugins/advanced-model-plugins.md | 52 ++++++++++ llm/__init__.py | 2 + llm/models.py | 118 ++++++++++++++++++++-- tests/test_tools.py | 134 +++++++++++++++++++++++++ 4 files changed, 299 insertions(+), 7 deletions(-) diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index fe494d88e..892ffe9fa 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -156,6 +156,58 @@ Here are the relevant dataclasses: .. autoclass:: llm.ToolResult ``` +(advanced-model-plugins-server-side-tools)= + +## Supporting server-side tools + +Server-side tools execute inside the provider's infrastructure rather than in LLM's local tool loop. Provider plugins represent these by subclassing {class}`llm.ServerSideTool`: + +```python +class ProviderSearch(llm.ServerSideTool): + "A search tool executed by Example Provider." + + name = "provider_search" + + def __init__(self, allowed_domains=None): + self.allowed_domains = allowed_domains + + def tool_spec(self, model): + spec = {"type": "provider_search"} + if self.allowed_domains: + spec["allowed_domains"] = self.allowed_domains + return spec + + def prepare_request(self, model, kwargs): + # Add any other request fields this tool requires. Merge with existing values instead of replacing them. + include = kwargs.setdefault("include", []) + if "provider_search.results" not in include: + include.append("provider_search.results") +``` + +The model classes that support the tool declare it explicitly: + +```python +class MyModel(llm.KeyModel): + server_side_tools = (ProviderSearch,) +``` + +This declaration is independent of `supports_tools`: that flag describes locally executed function tools. A model can support server-side tools, function tools, both or neither. + +`prompt.tools` can contain both {class}`llm.Tool` and {class}`llm.ServerSideTool` instances. The adapter should partition that list, put each server-side tool's `tool_spec(model)` result wherever its provider expects it and, after the baseline request is complete, call `tool.prepare_request(model, kwargs)` once for each server-side tool in list order. LLM validates the model declaration before `execute()` runs, but it does not interpret the specification or call either method itself. + +Providers with an OpenAI-compatible tools array can optionally support raw specifications by including `llm.ServerSideTool` in their declaration. This claims direct instances such as: + +```python +llm.ServerSideTool({"type": "browser_search"}) +``` + +Declaring the base class does not claim subclasses belonging to other providers. Unsupported combinations raise an error instead of silently dropping or serializing the tool as a function tool. Server-side tool calls returned by the provider should use the `server_executed=True` events described in {ref}`structured-messages-streaming`. + +```{eval-rst} +.. autoclass:: llm.ServerSideTool + :members: tool_spec, prepare_request +``` + (advanced-model-plugins-attachments)= diff --git a/llm/__init__.py b/llm/__init__.py index 18207550a..de994edb1 100644 --- a/llm/__init__.py +++ b/llm/__init__.py @@ -31,6 +31,7 @@ PauseChain, Prompt, Response, + ServerSideTool, Tool, Toolbox, ToolCall, @@ -68,6 +69,7 @@ "PauseChain", "Prompt", "Response", + "ServerSideTool", "Template", "Tool", "ToolCall", diff --git a/llm/models.py b/llm/models.py index 6fe1126e3..324cc2fde 100644 --- a/llm/models.py +++ b/llm/models.py @@ -187,6 +187,75 @@ def function(cls, function, name=None, description=None): ) +class ServerSideTool: + """A tool executed inside the provider's infrastructure. + + Instances are passed in ``tools=[...]`` alongside function tools. The + framework transports and validates them but never executes them. Provider + plugins subclass this class for their own tools; the base class can be + instantiated directly with a raw provider tool specification. + """ + + name: ClassVar[str] = "server_side_tool" + plugin: ClassVar[str | None] = None + implementation: ClassVar[None] = None + input_schema: ClassVar[dict] = {} + + def __init__(self, spec: dict | None = None): + self.spec = spec + self._config = {"spec": spec} + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + original_init = cls.__init__ + + @functools.wraps(original_init) + def wrapped_init(self, *args, **kwargs): + signature = inspect.signature(original_init) + bound = signature.bind(self, *args, **kwargs) + bound.apply_defaults() + original_init(self, *args, **kwargs) + self._config = { + name: value + for name, value in bound.arguments.items() + if name != "self" + and signature.parameters[name].kind + not in ( + inspect.Parameter.VAR_POSITIONAL, + inspect.Parameter.VAR_KEYWORD, + ) + } + + cls.__init__ = wrapped_init + + @property + def description(self) -> str | None: + return inspect.getdoc(self.__class__) + + def tool_spec(self, model) -> dict: + """Return this tool's provider-specific request specification.""" + if self.spec is None: + raise TypeError( + f"{self.__class__.__name__} does not define a raw provider tool spec" + ) + return self.spec + + def prepare_request(self, model, kwargs: dict) -> None: + """Add any other values this tool needs to provider request kwargs.""" + + def hash(self): + """Hash the definition separately from configured instances.""" + to_hash = { + "name": self.name, + "description": self.description, + "input_schema": self.input_schema, + "server_side": True, + } + if self.plugin: + to_hash["plugin"] = self.plugin + return hashlib.sha256(json.dumps(to_hash).encode("utf-8")).hexdigest() + + def _get_arguments_input_schema(function, name): signature = inspect.signature(function) type_hints = get_type_hints(function) @@ -366,7 +435,7 @@ class ToolOutput: attachments: list[Attachment] = field(default_factory=list) -ToolDef = Tool | Toolbox | Callable[..., Any] +ToolDef = Tool | Toolbox | ServerSideTool | Callable[..., Any] BeforeCallSync = Callable[[Tool | None, ToolCall], None] AfterCallSync = Callable[[Tool, ToolCall, ToolResult], None] BeforeCallAsync = Callable[[Tool | None, ToolCall], None | Awaitable[None]] @@ -415,7 +484,7 @@ class Prompt: system_fragments: list[str | Fragment] | None prompt_json: str | None schema: dict | type[BaseModel] | None - tools: list[Tool] + tools: list[Tool | ServerSideTool] tool_results: list[ToolResult] options: "Options" hide_reasoning: bool @@ -531,10 +600,10 @@ def messages(self): return result -def _wrap_tools(tools: list[ToolDef]) -> list[Tool]: +def _wrap_tools(tools: list[ToolDef]) -> list[Tool | ServerSideTool]: wrapped_tools = [] for tool in tools: - if isinstance(tool, Tool): + if isinstance(tool, (Tool, ServerSideTool)): wrapped_tools.append(tool) elif isinstance(tool, Toolbox): wrapped_tools.extend(tool.tools()) @@ -545,6 +614,35 @@ def _wrap_tools(tools: list[ToolDef]) -> list[Tool]: return wrapped_tools +def _partition_tools( + model: "_BaseModel", tools: Iterable[Tool | ServerSideTool] +) -> tuple[list[Tool], list[ServerSideTool]]: + """Partition tools and reject server-side tools the model did not claim.""" + function_tools = [] + server_side_tools = [] + declared = tuple(getattr(model, "server_side_tools", ())) + for tool in tools: + if isinstance(tool, ServerSideTool): + # Declaring ServerSideTool itself claims only direct raw-spec + # instances. Without this exact-type exception it would also + # accidentally claim every provider-specific subclass. + claimed = any( + type(tool) is candidate + if candidate is ServerSideTool + else isinstance(tool, candidate) + for candidate in declared + ) + if not claimed: + raise ValueError( + f"Model '{model.model_id}' does not support server-side tool " + f"'{tool.name}'. Run: llm tools -m {model.model_id}" + ) + server_side_tools.append(tool) + else: + function_tools.append(tool) + return function_tools, server_side_tools + + def _append_turn_input( chain: list[Any], prompt: str | None, @@ -1066,7 +1164,8 @@ def __init__( if self.prompt.schema and not self.model.supports_schema: raise ValueError(f"{self.model} does not support schemas") - if self.prompt.tools and not self.model.supports_tools: + function_tools, _ = _partition_tools(self.model, self.prompt.tools) + if function_tools and not self.model.supports_tools: raise ValueError(f"{self.model} does not support tools") def _messages_now(self) -> list[Any]: @@ -1790,7 +1889,9 @@ def execute_tool_calls( """ tool_results = [] effective_tools = _wrap_tools(tools) if tools is not None else self.prompt.tools - tools_by_name = {tool.name: tool for tool in effective_tools} + tools_by_name = { + tool.name: tool for tool in effective_tools if isinstance(tool, Tool) + } if tool_calls_list is None: tool_calls_list = self.tool_calls() @@ -2136,7 +2237,9 @@ async def execute_tool_calls( if tool_calls_list is None: tool_calls_list = await self.tool_calls() effective_tools = _wrap_tools(tools) if tools is not None else self.prompt.tools - tools_by_name = {tool.name: tool for tool in effective_tools} + tools_by_name = { + tool.name: tool for tool in effective_tools if isinstance(tool, Tool) + } # Run async prepare_async() on all Toolbox instances that need it instances_to_prepare: list[Toolbox] = [] @@ -3102,6 +3205,7 @@ class _BaseModel(ABC, _get_key_mixin): supports_schema = False supports_tools = False + server_side_tools: tuple[type[ServerSideTool], ...] = () class Options(_Options): pass diff --git a/tests/test_tools.py b/tests/test_tools.py index c1214e34d..40a5f9584 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -18,6 +18,140 @@ API_KEY = os.environ.get("PYTEST_OPENAI_API_KEY", None) or "badkey" +class DemoServerSideTool(llm.ServerSideTool): + "A server-side tool used by the core tests." + + name = "demo_server_tool" + + def __init__(self, value="demo"): + self.value = value + self.prepare_calls = 0 + + def tool_spec(self, model): + return {"type": self.name, "value": self.value} + + def prepare_request(self, model, kwargs): + self.prepare_calls += 1 + + +class ServerToolsOnlyModel(llm.Model): + model_id = "server-tools-only" + server_side_tools = (DemoServerSideTool,) + + def execute(self, prompt, stream, response, conversation): + yield "done" + + +class AsyncServerToolsOnlyModel(llm.AsyncModel): + model_id = "async-server-tools-only" + server_side_tools = (DemoServerSideTool,) + + async def execute(self, prompt, stream, response, conversation): + yield "done" + + +class MixedToolsModel(ServerToolsOnlyModel): + model_id = "mixed-tools" + supports_tools = True + + +class RawServerToolOnlyModel(ServerToolsOnlyModel): + model_id = "raw-server-tool-only" + server_side_tools = (llm.ServerSideTool,) + + +def test_server_side_tool_raw_spec_escape_hatch(): + tool = llm.ServerSideTool({"type": "browser_search"}) + model = ServerToolsOnlyModel() + + assert tool.tool_spec(model) == {"type": "browser_search"} + assert tool.name == "server_side_tool" + assert tool._config == {"spec": {"type": "browser_search"}} + + kwargs = {} + assert tool.prepare_request(model, kwargs) is None + assert kwargs == {} + + with pytest.raises(TypeError, match="raw provider tool spec"): + llm.ServerSideTool().tool_spec(model) + + +def test_declared_server_side_tool_does_not_require_function_tool_support(): + tool = DemoServerSideTool("one") + model = ServerToolsOnlyModel() + response = model.prompt("hello", tools=[tool]) + + assert response.text() == "done" + assert response.prompt.tools == [tool] + # Core transports and validates server-side tools but never invokes + # their provider request hook itself. + assert tool.prepare_calls == 0 + assert tool._config == {"value": "one"} + + +def test_unsupported_server_side_tool_fails_before_execution(mock_model): + with pytest.raises( + ValueError, + match=( + "Model 'mock' does not support server-side tool " + "'demo_server_tool'. Run: llm tools -m mock" + ), + ): + mock_model.prompt("hello", tools=[DemoServerSideTool()]) + + +def test_declaring_raw_escape_hatch_does_not_claim_every_subclass(): + model = RawServerToolOnlyModel() + assert model.prompt( + "hello", tools=[llm.ServerSideTool({"type": "custom"})] + ).text() == "done" + with pytest.raises(ValueError, match="does not support server-side tool"): + model.prompt("hello", tools=[DemoServerSideTool()]) + + +def test_function_tools_still_require_supports_tools(): + def local_tool(): + return "local" + + with pytest.raises(ValueError, match="does not support tools"): + ServerToolsOnlyModel().prompt("hello", tools=[local_tool]) + + +def test_local_executor_ignores_server_side_tools(): + def local_tool(): + return "local" + + tool = DemoServerSideTool() + response = MixedToolsModel().prompt( + "hello", tools=[tool, llm.Tool.function(local_tool)] + ) + response.text() + + results = response.execute_tool_calls( + tool_calls_list=[ + llm.ToolCall(name="local_tool", arguments={}), + llm.ToolCall(name=tool.name, arguments={}), + ] + ) + assert [result.output for result in results] == [ + "local", + 'Error: tool "demo_server_tool" does not exist', + ] + + +@pytest.mark.asyncio +async def test_async_declared_server_side_tool_and_executor_partition(): + tool = DemoServerSideTool() + model = AsyncServerToolsOnlyModel() + response = model.prompt("hello", tools=[tool]) + + assert await response.text() == "done" + results = await response.execute_tool_calls( + tool_calls_list=[llm.ToolCall(name=tool.name, arguments={})] + ) + assert results[0].output == 'Error: tool "demo_server_tool" does not exist' + + @pytest.mark.vcr def test_tool_use_basic(vcr): model = llm.get_model("gpt-4o-mini") From 70a6f45f41ab75ef506956b15c8796104cfc7893 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 18:03:01 -0700 Subject: [PATCH 229/258] Add OpenAI Code Interpreter server tool --- docs/openai-models.md | 39 ++++ docs/python-api.md | 21 ++ llm/default_plugins/openai_models.py | 109 +++++++--- tests/test_openai_responses.py | 290 +++++++++++++++++++++++++++ 4 files changed, 436 insertions(+), 23 deletions(-) diff --git a/docs/openai-models.md b/docs/openai-models.md index ed7b7f34d..ccfec1c37 100644 --- a/docs/openai-models.md +++ b/docs/openai-models.md @@ -83,6 +83,45 @@ The following features work with OpenAI models: - {ref}`Schemas ` can be used to influence the JSON structure of the model output. - {ref}`Model options ` can be used to set parameters like `temperature`. Use `llm models --options` for a full list of supported options. +(openai-models-code-interpreter)= + +## Code Interpreter + +Models that use the OpenAI Responses API can run Python in an OpenAI-managed container using the `CodeInterpreter` server-side tool: + +```python +import llm +from llm.default_plugins.openai_models import CodeInterpreter + +model = llm.get_model("gpt-5.6-luna") +response = model.prompt( + "Use the python tool to calculate 111111 * 333333", + tools=[CodeInterpreter()], +) +print(response.text()) +``` + +OpenAI calls this Code Interpreter, but models know it as the "python tool", so referring to that name in the prompt is the most explicit way to request it. + +By default OpenAI automatically creates a 1 GB container, or reuses an active container from the model's context. Configure a larger automatic container or make existing OpenAI files available to it like this: + +```python +CodeInterpreter( + memory_limit="4g", + file_ids=["file-1", "file-2"], +) +``` + +The accepted memory limits are `1g`, `4g`, `16g` and `64g`. Higher limits cost more. To reuse a container that was created separately, pass its ID: + +```python +CodeInterpreter(container="cntr_abc123") +``` + +An explicit container ID cannot be combined with `memory_limit` or `file_ids`. LLM automatically requests the full Code Interpreter output and records returned code and output as server-executed tool parts; it never tries to run that code locally. + +OpenAI containers are ephemeral and expire after 20 minutes without activity. See [OpenAI's Code Interpreter documentation](https://developers.openai.com/api/docs/guides/tools-code-interpreter) for container behavior, supported files and current pricing details. + (openai-models-service-tier)= ## Fast mode and service tiers diff --git a/docs/python-api.md b/docs/python-api.md index 88a614a4e..81e9f5def 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -310,6 +310,27 @@ def generate_image(prompt: str) -> llm.ToolOutput: .. autoclass:: llm.ToolOutput ``` +(python-api-server-side-tools)= + +#### Server-side tools + +Server-side tools are executed by the model provider during a response. Pass them in the same `tools=` list as function tools, but do not call `execute_tool_calls()` for them: their calls and results arrive in `response.messages()` with `server_executed=True` and are excluded from the local execution loop. + +For example, OpenAI Responses models support {ref}`Code Interpreter `: + +```python +import llm +from llm.default_plugins.openai_models import CodeInterpreter + +response = llm.get_model("gpt-5.6-luna").prompt( + "Use the python tool to calculate the first 20 Fibonacci numbers", + tools=[CodeInterpreter(memory_limit="4g")], +) +print(response.text()) +``` + +Provider plugins define these tools by subclassing {class}`llm.ServerSideTool`. A model rejects server-side tool classes it has not explicitly declared, rather than silently dropping them. See {ref}`advanced-model-plugins-server-side-tools` for the plugin API. + (python-api-toolbox)= #### Toolbox classes diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 634969320..d8115dbfe 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -4,7 +4,7 @@ import sys from collections.abc import AsyncGenerator, Iterable, Iterator from enum import Enum -from typing import Any, ClassVar +from typing import Any, ClassVar, Literal import click import httpx @@ -25,6 +25,7 @@ Response, hookimpl, ) +from llm.models import _partition_tools from llm.parts import StreamEvent from llm.utils import ( dicts_to_table_string, @@ -1604,9 +1605,60 @@ def _responses_attachment(attachment, image_detail=None): return {"type": "input_image", "image_url": url} +class CodeInterpreter(llm.ServerSideTool): + """Run Python in an OpenAI-managed container. + + With no ``container`` argument OpenAI creates or reuses an automatic + container. ``memory_limit`` and ``file_ids`` configure that automatic + container. Pass an existing ``cntr_`` ID as ``container`` to use it + explicitly instead. + """ + + name = "code_interpreter" + _memory_limits = frozenset({"1g", "4g", "16g", "64g"}) + + def __init__( + self, + container: str | None = None, + memory_limit: Literal["1g", "4g", "16g", "64g"] | None = None, + file_ids: list[str] | None = None, + ): + super().__init__() + if container is not None and not isinstance(container, str): + raise TypeError("container must be a string container ID") + if memory_limit is not None and memory_limit not in self._memory_limits: + raise ValueError( + "memory_limit must be one of: 1g, 4g, 16g or 64g" + ) + if container is not None and (memory_limit is not None or file_ids is not None): + raise ValueError( + "container cannot be combined with memory_limit or file_ids" + ) + self.container = container + self.memory_limit = memory_limit + self.file_ids = list(file_ids) if file_ids is not None else None + + def tool_spec(self, model): + if self.container is not None: + return {"type": "code_interpreter", "container": self.container} + container = {"type": "auto"} + if self.memory_limit is not None: + container["memory_limit"] = self.memory_limit + if self.file_ids is not None: + container["file_ids"] = list(self.file_ids) + return {"type": "code_interpreter", "container": container} + + def prepare_request(self, model, kwargs): + include = kwargs.setdefault("include", []) + if "code_interpreter_call.outputs" not in include: + include.append("code_interpreter_call.outputs") + + class _SharedResponses(_Shared): """Mixin that translates llm.Prompt into Responses API parameters.""" + server_side_tools = (CodeInterpreter, llm.ServerSideTool) + # Recurring boilerplate in Responses API payloads. Same contract as # _Shared.json_replacements, which this replaces for Responses # models: NEVER remove or change an existing entry - only ever @@ -1863,13 +1915,18 @@ def _build_responses_kwargs(self, prompt, stream): kwargs["text"] = text if prompt.tools: + _partition_tools(self, prompt.tools) kwargs["tools"] = [ - { - "type": "function", - "name": tool.name, - "description": tool.description or None, - "parameters": tool.input_schema, - } + ( + { + "type": "function", + "name": tool.name, + "description": tool.description or None, + "parameters": tool.input_schema, + } + if isinstance(tool, llm.Tool) + else tool.tool_spec(self) + ) for tool in prompt.tools ] @@ -1878,6 +1935,24 @@ def _build_responses_kwargs(self, prompt, stream): kwargs.update(opts) return kwargs + def _finalize_responses_kwargs(self, prompt, stream, instructions=None): + """Build complete request kwargs, then run server-tool hooks in order.""" + kwargs = self._build_responses_kwargs(prompt, stream) + if instructions is not None: + kwargs["instructions"] = instructions + kwargs["store"] = False + if self._reasoning and ( + self._reasoning_summary + or getattr(prompt.options, "reasoning_effort", None) + ): + include = kwargs.setdefault("include", []) + if "reasoning.encrypted_content" not in include: + include.append("reasoning.encrypted_content") + _, server_side_tools = _partition_tools(self, prompt.tools) + for tool in server_side_tools: + tool.prepare_request(self, kwargs) + return kwargs + def _set_usage_responses(self, response, usage): if not usage: return @@ -2201,6 +2276,7 @@ def execute( ) -> Iterator[str | StreamEvent]: if getattr(prompt.options, "chat_completions", None): chat = Chat(**self._delegate_chat_kwargs()) + _partition_tools(chat, prompt.tools) yield from chat.execute(prompt, stream, response, conversation, key) return @@ -2213,14 +2289,7 @@ def execute( input_items, instructions = self._build_responses_input( prompt, image_detail=image_detail ) - kwargs = self._build_responses_kwargs(prompt, stream) - if instructions is not None: - kwargs["instructions"] = instructions - kwargs["store"] = False - if self._reasoning and ( - self._reasoning_summary or getattr(prompt.options, "reasoning_effort", None) - ): - kwargs["include"] = ["reasoning.encrypted_content"] + kwargs = self._finalize_responses_kwargs(prompt, stream, instructions) client = self.get_client(key) usage = None @@ -2442,6 +2511,7 @@ async def execute( ) -> AsyncGenerator[str | StreamEvent, None]: if getattr(prompt.options, "chat_completions", None): chat = AsyncChat(**self._delegate_chat_kwargs()) + _partition_tools(chat, prompt.tools) async for event in chat.execute( prompt, stream, response, conversation, key ): @@ -2457,14 +2527,7 @@ async def execute( input_items, instructions = self._build_responses_input( prompt, image_detail=image_detail ) - kwargs = self._build_responses_kwargs(prompt, stream) - if instructions is not None: - kwargs["instructions"] = instructions - kwargs["store"] = False - if self._reasoning and ( - self._reasoning_summary or getattr(prompt.options, "reasoning_effort", None) - ): - kwargs["include"] = ["reasoning.encrypted_content"] + kwargs = self._finalize_responses_kwargs(prompt, stream, instructions) client = self.get_client(key, async_=True) usage = None diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index e0a4d1928..a6ea4d065 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -7,15 +7,281 @@ from pytest_httpx import IteratorStream import llm +from llm.default_plugins.openai_models import CodeInterpreter, Responses API_KEY = os.environ.get("PYTEST_OPENAI_API_KEY", None) or "badkey" +def _text_response_json(model="gpt-5.6-luna", text="ok"): + return { + "id": "resp_server_tool", + "object": "response", + "created_at": 1, + "model": model, + "output": [ + { + "type": "message", + "id": "msg_server_tool", + "role": "assistant", + "status": "completed", + "content": [ + {"type": "output_text", "text": text, "annotations": []} + ], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + "status": "completed", + } + + +@pytest.mark.parametrize( + ("tool", "expected"), + ( + ( + CodeInterpreter(), + {"type": "code_interpreter", "container": {"type": "auto"}}, + ), + ( + CodeInterpreter(memory_limit="4g", file_ids=["file-1", "file-2"]), + { + "type": "code_interpreter", + "container": { + "type": "auto", + "memory_limit": "4g", + "file_ids": ["file-1", "file-2"], + }, + }, + ), + ( + CodeInterpreter(container="cntr_123"), + {"type": "code_interpreter", "container": "cntr_123"}, + ), + ), +) +def test_code_interpreter_tool_spec(tool, expected): + assert tool.tool_spec(llm.get_model("gpt-5.6-luna")) == expected + + +def test_code_interpreter_validates_container_configuration(): + with pytest.raises(ValueError, match="memory_limit"): + CodeInterpreter(memory_limit="8g") + with pytest.raises(ValueError, match="cannot be combined"): + CodeInterpreter(container="cntr_123", memory_limit="4g") + with pytest.raises(ValueError, match="cannot be combined"): + CodeInterpreter(container="cntr_123", file_ids=["file-1"]) + with pytest.raises(TypeError, match="container must be a string"): + CodeInterpreter(container={"type": "auto"}) + + +def test_code_interpreter_prepare_request_is_additive_and_idempotent(): + tool = CodeInterpreter() + kwargs = {"include": ["reasoning.encrypted_content"]} + + tool.prepare_request(llm.get_model("gpt-5.6-luna"), kwargs) + tool.prepare_request(llm.get_model("gpt-5.6-luna"), kwargs) + + assert kwargs == { + "include": [ + "reasoning.encrypted_content", + "code_interpreter_call.outputs", + ] + } + + +def test_server_side_prepare_request_runs_in_list_order_after_baseline(): + class IncludeMarker(CodeInterpreter): + def __init__(self, marker): + super().__init__() + self.marker = marker + + def tool_spec(self, model): + return {"type": f"marker_{self.marker}"} + + def prepare_request(self, model, kwargs): + assert kwargs["store"] is False + assert len(kwargs["tools"]) == 2 + kwargs.setdefault("include", []).append(self.marker) + + model = llm.get_model("gpt-5.6-luna") + + class FakePrompt: + pass + + prompt = FakePrompt() + prompt.options = model.Options() + prompt.tools = [IncludeMarker("first"), IncludeMarker("second")] + prompt.schema = None + prompt.hide_reasoning = False + + kwargs = model._finalize_responses_kwargs( + prompt, stream=False, instructions="Be useful" + ) + assert kwargs["instructions"] == "Be useful" + assert kwargs["include"] == [ + "reasoning.encrypted_content", + "first", + "second", + ] + + +def test_responses_mixes_function_and_code_interpreter_tools(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json=_text_response_json(), + headers={"Content-Type": "application/json"}, + ) + + def multiply(a: int, b: int) -> int: + return a * b + + model = llm.get_model("gpt-5.6-luna") + response = model.prompt( + "Calculate 111 * 333 using the python tool", + tools=[multiply, CodeInterpreter(memory_limit="4g")], + stream=False, + key="test", + ) + assert response.text() == "ok" + + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["tools"][0]["type"] == "function" + assert request_body["tools"][0]["name"] == "multiply" + assert request_body["tools"][1] == { + "type": "code_interpreter", + "container": {"type": "auto", "memory_limit": "4g"}, + } + assert request_body["include"] == [ + "reasoning.encrypted_content", + "code_interpreter_call.outputs", + ] + + +def test_responses_raw_server_tool_passthrough_on_custom_endpoint(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://example.test/v1/responses", + json=_text_response_json(model="custom-model"), + headers={"Content-Type": "application/json"}, + ) + model = Responses( + "custom-model", + api_base="https://example.test/v1", + supports_tools=False, + ) + raw_spec = {"type": "browser_search", "depth": "deep"} + response = model.prompt( + "Search", tools=[llm.ServerSideTool(raw_spec)], stream=False, key="test" + ) + + assert response.text() == "ok" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["tools"] == [raw_spec] + + +def test_code_interpreter_rejected_by_chat_and_chat_fallback(): + from llm.default_plugins.openai_models import Chat + + chat = Chat("chat-model", supports_tools=True) + with pytest.raises(ValueError, match="llm tools -m chat-model"): + chat.prompt("Calculate", tools=[CodeInterpreter()]) + + responses_model = llm.get_model("gpt-5.6-luna") + response = responses_model.prompt( + "Calculate", tools=[CodeInterpreter()], chat_completions=True, key="test" + ) + with pytest.raises(ValueError, match="llm tools -m gpt-5.6-luna"): + response.text() + + +@pytest.mark.asyncio +async def test_async_responses_code_interpreter_request(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json=_text_response_json(), + headers={"Content-Type": "application/json"}, + ) + model = llm.get_async_model("gpt-5.6-luna") + response = model.prompt( + "Calculate", + tools=[CodeInterpreter(file_ids=["file-1"])], + stream=False, + key="test", + ) + + assert await response.text() == "ok" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["tools"] == [ + { + "type": "code_interpreter", + "container": {"type": "auto", "file_ids": ["file-1"]}, + } + ] + assert request_body["include"] == [ + "reasoning.encrypted_content", + "code_interpreter_call.outputs", + ] + + def _responses_sse(event_type, data): data = {"type": event_type, **data} return f"event: {event_type}\ndata: {json.dumps(data)}\n\n".encode() +def _code_interpreter_stream(): + yield _responses_sse( + "response.output_item.added", + { + "output_index": 0, + "item": { + "id": "ci_stream", + "type": "code_interpreter_call", + "status": "in_progress", + "container_id": "cntr_stream", + "code": "", + "outputs": [], + }, + }, + ) + yield _responses_sse( + "response.output_item.done", + { + "output_index": 0, + "item": { + "id": "ci_stream", + "type": "code_interpreter_call", + "status": "completed", + "container_id": "cntr_stream", + "code": "print(6 * 7)", + "outputs": [{"type": "logs", "logs": "42\n"}], + }, + }, + ) + yield _responses_sse( + "response.output_item.added", + { + "output_index": 1, + "item": { + "id": "msg_stream", + "type": "message", + "status": "in_progress", + "role": "assistant", + "content": [], + }, + }, + ) + yield _responses_sse( + "response.output_text.delta", + { + "item_id": "msg_stream", + "output_index": 1, + "content_index": 0, + "delta": "42", + }, + ) + + def _responses_reasoning_summary_stream(): yield _responses_sse( "response.reasoning_summary_text.delta", @@ -929,6 +1195,30 @@ def test_code_interpreter_multi_message_response(httpx_mock): assert response.tool_calls() == [] +def test_code_interpreter_streaming_output_and_request(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + stream=IteratorStream(_code_interpreter_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + model = llm.get_model("gpt-5.6-luna") + response = model.prompt("Calculate", tools=[CodeInterpreter()], key="test") + + events = list(response.stream_events()) + assert [(event.type, event.chunk) for event in events] == [ + ("tool_call_name", "code_interpreter"), + ("tool_call_args", json.dumps({"code": "print(6 * 7)"})), + ("tool_result", "42\n"), + ("text", "42"), + ] + assert all(event.server_executed for event in events[:3]) + assert response.tool_calls() == [] + + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert "code_interpreter_call.outputs" in request_body["include"] + + def test_server_tool_parts_not_replayed_as_function_calls(): from llm.parts import Message, TextPart, ToolCallPart, ToolResultPart From 73295a9f520283173bcc48a4f2cec4740aa2b39c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 18:49:14 -0700 Subject: [PATCH 230/258] Resolve server-side tools from model instances --- docs/openai-models.md | 6 +++ docs/plugins/advanced-model-plugins.md | 10 +++-- docs/python-api.md | 2 +- llm/cli.py | 39 +++++++++++++----- llm/default_plugins/openai_models.py | 4 +- llm/models.py | 8 +++- tests/test_cli_openai_models.py | 57 ++++++++++++++++++++++++++ tests/test_tools.py | 31 ++++++++++++-- 8 files changed, 135 insertions(+), 22 deletions(-) diff --git a/docs/openai-models.md b/docs/openai-models.md index ccfec1c37..d290a9ce2 100644 --- a/docs/openai-models.md +++ b/docs/openai-models.md @@ -89,6 +89,12 @@ The following features work with OpenAI models: Models that use the OpenAI Responses API can run Python in an OpenAI-managed container using the `CodeInterpreter` server-side tool: +```bash +llm -m gpt-5.6-luna -T 'CodeInterpreter(memory_limit="4g")' 'Run this calculation' +``` + +The same tool can be used from Python: + ```python import llm from llm.default_plugins.openai_models import CodeInterpreter diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index 892ffe9fa..d2d5a94eb 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -184,18 +184,20 @@ class ProviderSearch(llm.ServerSideTool): include.append("provider_search.results") ``` -The model classes that support the tool declare it explicitly: +Model instances expose the server-side tool classes they support using the `supported_server_side_tools` property: ```python class MyModel(llm.KeyModel): - server_side_tools = (ProviderSearch,) + @property + def supported_server_side_tools(self): + return (ProviderSearch,) ``` -This declaration is independent of `supports_tools`: that flag describes locally executed function tools. A model can support server-side tools, function tools, both or neither. +This property is evaluated on the model instance, so it can use `self.model_id` or other instance configuration to vary the available tools. It is independent of `supports_tools`: that flag describes locally executed function tools. A model can support server-side tools, function tools, both or neither. `prompt.tools` can contain both {class}`llm.Tool` and {class}`llm.ServerSideTool` instances. The adapter should partition that list, put each server-side tool's `tool_spec(model)` result wherever its provider expects it and, after the baseline request is complete, call `tool.prepare_request(model, kwargs)` once for each server-side tool in list order. LLM validates the model declaration before `execute()` runs, but it does not interpret the specification or call either method itself. -Providers with an OpenAI-compatible tools array can optionally support raw specifications by including `llm.ServerSideTool` in their declaration. This claims direct instances such as: +Providers with an OpenAI-compatible tools array can optionally support raw specifications by including `llm.ServerSideTool` in this property. This claims direct instances such as: ```python llm.ServerSideTool({"type": "browser_search"}) diff --git a/docs/python-api.md b/docs/python-api.md index 81e9f5def..8f4430c90 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -329,7 +329,7 @@ response = llm.get_model("gpt-5.6-luna").prompt( print(response.text()) ``` -Provider plugins define these tools by subclassing {class}`llm.ServerSideTool`. A model rejects server-side tool classes it has not explicitly declared, rather than silently dropping them. See {ref}`advanced-model-plugins-server-side-tools` for the plugin API. +Provider plugins define these tools by subclassing {class}`llm.ServerSideTool`. Each model instance returns its supported classes from `supported_server_side_tools`; a model rejects classes it has not explicitly included rather than silently dropping them. See {ref}`advanced-model-plugins-server-side-tools` for the plugin API. (python-api-toolbox)= diff --git a/llm/cli.py b/llm/cli.py index f05b990df..8c09488d7 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -37,6 +37,7 @@ Fragment, KeyModel, Response, + ServerSideTool, Template, Tool, Toolbox, @@ -1016,7 +1017,7 @@ def read_prompt(): prompt_method = conversation.prompt tool_kwargs = _tool_chain_kwargs( - tools, python_tools, tools_debug, tools_approve, chain_limit + tools, python_tools, tools_debug, tools_approve, chain_limit, model=model ) if tool_kwargs: prompt_method = conversation.chain @@ -1292,7 +1293,14 @@ def chat( kwargs["options"] = validated_options kwargs.update( - _tool_chain_kwargs(tools, python_tools, tools_debug, tools_approve, chain_limit) + _tool_chain_kwargs( + tools, + python_tools, + tools_debug, + tools_approve, + chain_limit, + model=model, + ) ) should_stream = model.can_stream and not no_stream @@ -4158,29 +4166,38 @@ def _approve_tool_call(_, tool_call): def _gather_tools( - tool_specs: list[str], python_tools: list[str] -) -> list[Tool | type[Toolbox]]: - tools: list[Tool | type[Toolbox]] = [] + tool_specs: list[str], python_tools: list[str], model=None +) -> list[Tool | Toolbox | ServerSideTool]: + tools: list[Tool | Toolbox | ServerSideTool] = [] if python_tools: for code_or_path in python_tools: tools.extend(_tools_from_code(code_or_path)) registered_tools = get_tools() + server_side_tool_classes = { + tool_class.__name__: tool_class + for tool_class in ( + model.supported_server_side_tools if model is not None else () + ) + } + available_tools = {**registered_tools, **server_side_tool_classes} registered_classes = { - key: value for key, value in registered_tools.items() if inspect.isclass(value) + key: value for key, value in available_tools.items() if inspect.isclass(value) } bad_tools = [ - tool for tool in tool_specs if tool.split("(")[0] not in registered_tools + tool + for tool in tool_specs + if tool.split("(", 1)[0].strip() not in available_tools ] if bad_tools: raise click.ClickException( "Tool(s) {} not found. Available tools: {}".format( - ", ".join(bad_tools), ", ".join(registered_tools.keys()) + ", ".join(bad_tools), ", ".join(available_tools.keys()) ) ) for tool_spec in tool_specs: if not tool_spec[0].isupper(): # It's a function - tools.append(registered_tools[tool_spec]) + tools.append(available_tools[tool_spec]) else: # It's a class tools.append(instantiate_from_spec(registered_classes, tool_spec)) @@ -4188,10 +4205,10 @@ def _gather_tools( def _tool_chain_kwargs( - tool_specs, python_tools, tools_debug, tools_approve, chain_limit + tool_specs, python_tools, tools_debug, tools_approve, chain_limit, model=None ): """Build Conversation.chain() keyword arguments for CLI-selected tools.""" - tool_implementations = _gather_tools(tool_specs, python_tools) + tool_implementations = _gather_tools(tool_specs, python_tools, model=model) if not tool_implementations: return {} kwargs = { diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index d8115dbfe..2e676abc1 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1657,7 +1657,9 @@ def prepare_request(self, model, kwargs): class _SharedResponses(_Shared): """Mixin that translates llm.Prompt into Responses API parameters.""" - server_side_tools = (CodeInterpreter, llm.ServerSideTool) + @property + def supported_server_side_tools(self): + return (CodeInterpreter, llm.ServerSideTool) # Recurring boilerplate in Responses API payloads. Same contract as # _Shared.json_replacements, which this replaces for Responses diff --git a/llm/models.py b/llm/models.py index 324cc2fde..1c7657f9d 100644 --- a/llm/models.py +++ b/llm/models.py @@ -620,7 +620,7 @@ def _partition_tools( """Partition tools and reject server-side tools the model did not claim.""" function_tools = [] server_side_tools = [] - declared = tuple(getattr(model, "server_side_tools", ())) + declared = tuple(model.supported_server_side_tools) for tool in tools: if isinstance(tool, ServerSideTool): # Declaring ServerSideTool itself claims only direct raw-spec @@ -3205,7 +3205,11 @@ class _BaseModel(ABC, _get_key_mixin): supports_schema = False supports_tools = False - server_side_tools: tuple[type[ServerSideTool], ...] = () + + @property + def supported_server_side_tools(self) -> tuple[type[ServerSideTool], ...]: + """Server-side tool classes accepted by this model instance.""" + return () class Options(_Options): pass diff --git a/tests/test_cli_openai_models.py b/tests/test_cli_openai_models.py index ac37c0db0..95a7043c9 100644 --- a/tests/test_cli_openai_models.py +++ b/tests/test_cli_openai_models.py @@ -214,6 +214,63 @@ def test_gpt5_verbosity_option_validates_allowed_values(): assert "Input should be 'low', 'medium' or 'high'" in result.output +def test_code_interpreter_cli_tool_is_resolved_from_model(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_code_interpreter_cli", + "object": "response", + "created_at": 1, + "model": "gpt-5.6-luna", + "output": [ + { + "type": "message", + "id": "msg_code_interpreter_cli", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Calculated", + "annotations": [], + } + ], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + result = CliRunner().invoke( + cli, + [ + "-m", + "gpt-5.6-luna", + "-T", + 'CodeInterpreter(memory_limit="4g")', + "--no-stream", + "--no-log", + "--key", + "x", + "Run this calculation", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.stdout == "Calculated\n" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["tools"] == [ + { + "type": "code_interpreter", + "container": {"type": "auto", "memory_limit": "4g"}, + } + ] + assert "code_interpreter_call.outputs" in request_body["include"] + + @pytest.mark.parametrize( "model_id,expected_description", ( diff --git a/tests/test_tools.py b/tests/test_tools.py index 40a5f9584..1597a06e2 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -36,7 +36,10 @@ def prepare_request(self, model, kwargs): class ServerToolsOnlyModel(llm.Model): model_id = "server-tools-only" - server_side_tools = (DemoServerSideTool,) + + @property + def supported_server_side_tools(self): + return (DemoServerSideTool,) def execute(self, prompt, stream, response, conversation): yield "done" @@ -44,7 +47,10 @@ def execute(self, prompt, stream, response, conversation): class AsyncServerToolsOnlyModel(llm.AsyncModel): model_id = "async-server-tools-only" - server_side_tools = (DemoServerSideTool,) + + @property + def supported_server_side_tools(self): + return (DemoServerSideTool,) async def execute(self, prompt, stream, response, conversation): yield "done" @@ -57,7 +63,10 @@ class MixedToolsModel(ServerToolsOnlyModel): class RawServerToolOnlyModel(ServerToolsOnlyModel): model_id = "raw-server-tool-only" - server_side_tools = (llm.ServerSideTool,) + + @property + def supported_server_side_tools(self): + return (llm.ServerSideTool,) def test_server_side_tool_raw_spec_escape_hatch(): @@ -109,6 +118,22 @@ def test_declaring_raw_escape_hatch_does_not_claim_every_subclass(): model.prompt("hello", tools=[DemoServerSideTool()]) +def test_server_side_tool_support_can_vary_by_model_instance(): + class ConditionalModel(ServerToolsOnlyModel): + def __init__(self, enabled): + self.enabled = enabled + + @property + def supported_server_side_tools(self): + return (DemoServerSideTool,) if self.enabled else () + + assert ConditionalModel(True).prompt( + "hello", tools=[DemoServerSideTool()] + ).text() == "done" + with pytest.raises(ValueError, match="does not support server-side tool"): + ConditionalModel(False).prompt("hello", tools=[DemoServerSideTool()]) + + def test_function_tools_still_require_supports_tools(): def local_tool(): return "local" From 0e2aa5afbbb552f50c3298213ea69dc712d276a1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 19:02:53 -0700 Subject: [PATCH 231/258] List server-side tools by model --- docs/usage.md | 8 ++++++ llm/cli.py | 50 +++++++++++++++++++++++++++++---- tests/test_cli_openai_models.py | 50 +++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 6 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index ff55e5d03..d8f88b64b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -171,6 +171,14 @@ Run this command to see a list of available tools from plugins: ```bash llm tools ``` +Server-side tools are model-specific. Pass `-m/--model` to include the tools supported by a particular model: + +```bash +llm tools -m gpt-5.6-luna +``` + +These are displayed in a separate `Server-side tools` section with their constructor signatures and documentation. Add `--json` to return them in a `server_side_tools` array whose entries have `"server_side": true`. + If you run a prompt that uses tools from plugins (as opposed to tools provided using the `--functions` option) continuing that conversation using `llm -c` will reuse the tools from the first prompt. Running `llm chat -c` will start a chat that continues using those same tools. For example: ``` diff --git a/llm/cli.py b/llm/cli.py index 8c09488d7..c07a87e66 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -2689,14 +2689,38 @@ def tools(): @tools.command(name="list") @click.argument("tool_defs", nargs=-1) @click.option("json_", "--json", is_flag=True, help="Output as JSON") +@click.option("model_id", "-m", "--model", help="List tools supported by this model") @click.option( "python_tools", "--functions", help="Python code block or file path defining functions to register as tools", multiple=True, ) -def tools_list(tool_defs, json_, python_tools): - "List available tools that have been provided by plugins" +def tools_list(tool_defs, json_, model_id, python_tools): + "List available tools, optionally including tools supported by a model" + + model = None + if model_id: + try: + model = get_model(model_id) + except UnknownModelError as ex: + raise click.ClickException(str(ex)) + + server_side_tools = [] + if model is not None: + for tool_class in model.supported_server_side_tools: + try: + signature = str(inspect.signature(tool_class)) + except (ValueError, TypeError): + signature = "(...)" + server_side_tools.append( + { + "name": tool_class.__name__, + "description": inspect.getdoc(tool_class), + "signature": signature, + "server_side": True, + } + ) def introspect_tools(toolbox): # Instances report their tools(), which may be generated dynamically. @@ -2780,11 +2804,11 @@ def introspect_tools(toolbox): } ) if json_: + output = {"tools": output_tools, "toolboxes": output_toolboxes} + if model is not None: + output["server_side_tools"] = server_side_tools click.echo( - json.dumps( - {"tools": output_tools, "toolboxes": output_toolboxes}, - indent=2, - ) + json.dumps(output, indent=2) ) else: for tool in tool_objects: @@ -2834,6 +2858,20 @@ def introspect_tools(toolbox): click.echo( textwrap.indent(tool_info["description"].strip(), " ") + "\n" ) + if model is not None: + if server_side_tools: + click.echo( + f"Server-side tools for {model.model_id} " + "(executed by the provider):\n" + ) + for tool_info in server_side_tools: + click.echo(f"{tool_info['name']}{tool_info['signature']}\n") + if tool_info["description"]: + click.echo( + textwrap.indent(tool_info["description"], " ") + "\n" + ) + else: + click.echo(f"No server-side tools for {model.model_id}.") @cli.group( diff --git a/tests/test_cli_openai_models.py b/tests/test_cli_openai_models.py index 95a7043c9..ecf6fc3d0 100644 --- a/tests/test_cli_openai_models.py +++ b/tests/test_cli_openai_models.py @@ -271,6 +271,56 @@ def test_code_interpreter_cli_tool_is_resolved_from_model(httpx_mock): assert "code_interpreter_call.outputs" in request_body["include"] +def test_tools_list_for_model_includes_server_side_tools(): + runner = CliRunner() + result = runner.invoke(cli, ["tools", "-m", "gpt-5.6-luna"]) + + assert result.exit_code == 0 + assert ( + "Server-side tools for gpt-5.6-luna (executed by the provider):\n" + in result.output + ) + assert "CodeInterpreter(" in result.output + assert "memory_limit:" in result.output + assert "Literal['1g', '4g', '16g', '64g']" in result.output + assert "Run Python in an OpenAI-managed container." in result.output + assert "ServerSideTool(spec: dict | None = None)" in result.output + + json_result = runner.invoke( + cli, ["tools", "-m", "gpt-5.6-luna", "--json"] + ) + assert json_result.exit_code == 0 + server_side_tools = json.loads(json_result.output)["server_side_tools"] + assert [tool["name"] for tool in server_side_tools] == [ + "CodeInterpreter", + "ServerSideTool", + ] + assert all(tool["server_side"] is True for tool in server_side_tools) + assert server_side_tools[0]["signature"].startswith("(container:") + assert server_side_tools[0]["description"].startswith( + "Run Python in an OpenAI-managed container." + ) + + +def test_tools_list_for_model_with_no_server_side_tools(): + runner = CliRunner() + result = runner.invoke(cli, ["tools", "-m", "chatgpt"]) + + assert result.exit_code == 0 + assert "No server-side tools for gpt-3.5-turbo.\n" in result.output + + json_result = runner.invoke(cli, ["tools", "-m", "chatgpt", "--json"]) + assert json_result.exit_code == 0 + assert json.loads(json_result.output)["server_side_tools"] == [] + + +def test_tools_list_rejects_unknown_model(): + result = CliRunner().invoke(cli, ["tools", "-m", "not-a-model"]) + + assert result.exit_code == 1 + assert "Unknown model: not-a-model" in result.output + + @pytest.mark.parametrize( "model_id,expected_description", ( From 317a922e30ea204781ba9d03438d6c8f2f1d9b9e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 19:20:28 -0700 Subject: [PATCH 232/258] Add OpenAI Web Search server tool --- docs/openai-models.md | 50 +++++++ docs/python-api.md | 2 +- llm/default_plugins/openai_models.py | 161 ++++++++++++++++++++++- tests/test_cli_openai_models.py | 65 ++++++++- tests/test_openai_responses.py | 190 ++++++++++++++++++++++++++- 5 files changed, 456 insertions(+), 12 deletions(-) diff --git a/docs/openai-models.md b/docs/openai-models.md index d290a9ce2..13dfa19cd 100644 --- a/docs/openai-models.md +++ b/docs/openai-models.md @@ -83,6 +83,56 @@ The following features work with OpenAI models: - {ref}`Schemas ` can be used to influence the JSON structure of the model output. - {ref}`Model options ` can be used to set parameters like `temperature`. Use `llm models --options` for a full list of supported options. +(openai-models-web-search)= + +## Web Search + +Models that use the OpenAI Responses API can search the web using the `WebSearch` server-side tool: + +```bash +llm -m gpt-5.6-luna -T WebSearch 'Search the web for a positive news story from today' +``` + +The model decides whether to search based on the prompt. The Python API accepts the same tool: + +```python +import llm +from llm.default_plugins.openai_models import WebSearch + +response = llm.get_model("gpt-5.6-luna").prompt( + "Search the web for a positive news story from today", + tools=[WebSearch(include_sources=True)], +) +print(response.text()) +``` + +Domain filters accept up to 100 allowed domains and up to 100 blocked domains. Omit `http://` or `https://`; each entry also covers its subdomains: + +```python +WebSearch( + allowed_domains=["openai.com", "python.org"], + blocked_domains=["example.com"], +) +``` + +Use `search_context_size="low"`, `"medium"` or `"high"` to control how much search-result context is made available to the model. Set `external_web_access=False` for cached/indexed results only. `return_token_budget="unlimited"` removes the standard returned-token limit for longer GPT-5 reasoning searches and can increase latency and cost. + +Approximate geographic context can include a two-letter country code, city, region and IANA timezone. LLM adds the required `"type": "approximate"` field when it is omitted: + +```python +WebSearch( + user_location={ + "country": "GB", + "city": "London", + "timezone": "Europe/London", + } +) +``` + +`include_sources=True` requests every URL consulted during search and records them in the server-executed tool call arguments. For image search, use `search_content_types=["image", "text"]` and `image_settings={"max_results": 3, "caption": True}`. Set `include_results=True` to request and record the raw image result objects in the corresponding server-executed tool result. + +See [OpenAI's Web Search documentation](https://developers.openai.com/api/docs/guides/tools-web-search) for model behavior, current limitations, citations and pricing details. + (openai-models-code-interpreter)= ## Code Interpreter diff --git a/docs/python-api.md b/docs/python-api.md index 8f4430c90..33807d2a7 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -316,7 +316,7 @@ def generate_image(prompt: str) -> llm.ToolOutput: Server-side tools are executed by the model provider during a response. Pass them in the same `tools=` list as function tools, but do not call `execute_tool_calls()` for them: their calls and results arrive in `response.messages()` with `server_executed=True` and are excluded from the local execution loop. -For example, OpenAI Responses models support {ref}`Code Interpreter `: +For example, OpenAI Responses models support {ref}`Web Search ` and {ref}`Code Interpreter `: ```python import llm diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 2e676abc1..c6180bb53 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1605,6 +1605,151 @@ def _responses_attachment(attachment, image_detail=None): return {"type": "input_image", "image_url": url} +class WebSearch(llm.ServerSideTool): + """Search the web using OpenAI's hosted search tool. + + Configure domain filters, approximate location, result context, live web + access and image search through constructor arguments. Set + ``include_sources`` to retain every consulted URL or ``include_results`` + to retain raw results such as image search results. + """ + + name = "web_search" + _search_context_sizes = frozenset({"low", "medium", "high"}) + _return_token_budgets = frozenset({"default", "unlimited"}) + _search_content_types = frozenset({"text", "image"}) + + def __init__( + self, + allowed_domains: list[str] | None = None, + blocked_domains: list[str] | None = None, + user_location: dict | None = None, + search_context_size: Literal["low", "medium", "high"] | None = None, + external_web_access: bool | None = None, + return_token_budget: Literal["default", "unlimited"] | None = None, + search_content_types: list[Literal["text", "image"]] | None = None, + image_settings: dict | None = None, + include_sources: bool = False, + include_results: bool = False, + ): + super().__init__() + self.allowed_domains = self._validate_domains( + "allowed_domains", allowed_domains + ) + self.blocked_domains = self._validate_domains( + "blocked_domains", blocked_domains + ) + if ( + search_context_size is not None + and search_context_size not in self._search_context_sizes + ): + raise ValueError("search_context_size must be one of: low, medium or high") + if external_web_access is not None and not isinstance( + external_web_access, bool + ): + raise TypeError("external_web_access must be a boolean") + if ( + return_token_budget is not None + and return_token_budget not in self._return_token_budgets + ): + raise ValueError("return_token_budget must be default or unlimited") + if search_content_types is not None: + if not isinstance(search_content_types, list): + raise TypeError("search_content_types must be a list") + invalid_content_types = set(search_content_types).difference( + self._search_content_types + ) + if invalid_content_types: + raise ValueError("search_content_types must contain text and/or image") + if user_location is not None: + if not isinstance(user_location, dict): + raise TypeError("user_location must be a dictionary") + user_location = dict(user_location) + user_location.setdefault("type", "approximate") + if user_location["type"] != "approximate": + raise ValueError("user_location type must be approximate") + if image_settings is not None: + if not isinstance(image_settings, dict): + raise TypeError("image_settings must be a dictionary") + image_settings = dict(image_settings) + max_results = image_settings.get("max_results") + if max_results is not None and ( + isinstance(max_results, bool) + or not isinstance(max_results, int) + or max_results < 1 + ): + raise ValueError( + "image_settings max_results must be a positive integer" + ) + caption = image_settings.get("caption") + if caption is not None and not isinstance(caption, bool): + raise TypeError("image_settings caption must be a boolean") + if not isinstance(include_sources, bool): + raise TypeError("include_sources must be a boolean") + if not isinstance(include_results, bool): + raise TypeError("include_results must be a boolean") + self.user_location = user_location + self.search_context_size = search_context_size + self.external_web_access = external_web_access + self.return_token_budget = return_token_budget + self.search_content_types = ( + list(search_content_types) if search_content_types is not None else None + ) + self.image_settings = image_settings + self.include_sources = include_sources + self.include_results = include_results + + @staticmethod + def _validate_domains(name, domains): + if domains is None: + return None + if not isinstance(domains, list): + raise TypeError(f"{name} must be a list") + if len(domains) > 100: + raise ValueError(f"{name} cannot contain more than 100 domains") + for domain in domains: + if not isinstance(domain, str) or not domain: + raise TypeError(f"{name} entries must be non-empty strings") + if domain.lower().startswith(("http://", "https://")): + raise ValueError(f"{name} entries must omit the URL scheme") + return list(domains) + + def tool_spec(self, model): + spec = {"type": "web_search"} + if self.allowed_domains is not None or self.blocked_domains is not None: + filters = {} + if self.allowed_domains is not None: + filters["allowed_domains"] = list(self.allowed_domains) + if self.blocked_domains is not None: + filters["blocked_domains"] = list(self.blocked_domains) + spec["filters"] = filters + for key in ( + "user_location", + "search_context_size", + "external_web_access", + "return_token_budget", + "search_content_types", + "image_settings", + ): + value = getattr(self, key) + if value is not None: + if isinstance(value, dict): + value = dict(value) + elif isinstance(value, list): + value = list(value) + spec[key] = value + return spec + + def prepare_request(self, model, kwargs): + if not self.include_sources and not self.include_results: + return + include = kwargs.setdefault("include", []) + if self.include_sources and "web_search_call.action.sources" not in include: + include.append("web_search_call.action.sources") + if self.include_results and "web_search_call.results" not in include: + include.append("web_search_call.results") + + class CodeInterpreter(llm.ServerSideTool): """Run Python in an OpenAI-managed container. @@ -1659,7 +1804,7 @@ class _SharedResponses(_Shared): @property def supported_server_side_tools(self): - return (CodeInterpreter, llm.ServerSideTool) + return (WebSearch, CodeInterpreter, llm.ServerSideTool) # Recurring boilerplate in Responses API payloads. Same contract as # _Shared.json_replacements, which this replaces for Responses @@ -2085,13 +2230,19 @@ def _server_tool_events(self, item, message_index): message_index=message_index, ) ) - # Search results are not included in the call item - they - # surface as citations in the following message - so the - # result records completion status only. + results = getattr(item, "results", None) or [] + results = [ + result.model_dump() if hasattr(result, "model_dump") else result + for result in results + ] events.append( StreamEvent( type="tool_result", - chunk=getattr(item, "status", None) or "completed", + chunk=( + json.dumps(results) + if results + else (getattr(item, "status", None) or "completed") + ), tool_call_id=item_id, server_executed=True, tool_name="web_search", diff --git a/tests/test_cli_openai_models.py b/tests/test_cli_openai_models.py index ecf6fc3d0..b550d32a1 100644 --- a/tests/test_cli_openai_models.py +++ b/tests/test_cli_openai_models.py @@ -271,6 +271,64 @@ def test_code_interpreter_cli_tool_is_resolved_from_model(httpx_mock): assert "code_interpreter_call.outputs" in request_body["include"] +def test_web_search_cli_tool_is_resolved_from_model(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_web_search_cli", + "object": "response", + "created_at": 1, + "model": "gpt-5.6-luna", + "output": [ + { + "type": "message", + "id": "msg_web_search_cli", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Search complete", + "annotations": [], + } + ], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + result = CliRunner().invoke( + cli, + [ + "-m", + "gpt-5.6-luna", + "-T", + 'WebSearch(allowed_domains=["openai.com"], search_context_size="low", include_sources=true)', + "--no-stream", + "--no-log", + "--key", + "x", + "Search for OpenAI news", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.stdout == "Search complete\n" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["tools"] == [ + { + "type": "web_search", + "filters": {"allowed_domains": ["openai.com"]}, + "search_context_size": "low", + } + ] + assert "web_search_call.action.sources" in request_body["include"] + + def test_tools_list_for_model_includes_server_side_tools(): runner = CliRunner() result = runner.invoke(cli, ["tools", "-m", "gpt-5.6-luna"]) @@ -281,6 +339,8 @@ def test_tools_list_for_model_includes_server_side_tools(): in result.output ) assert "CodeInterpreter(" in result.output + assert "WebSearch(" in result.output + assert "allowed_domains:" in result.output assert "memory_limit:" in result.output assert "Literal['1g', '4g', '16g', '64g']" in result.output assert "Run Python in an OpenAI-managed container." in result.output @@ -292,13 +352,14 @@ def test_tools_list_for_model_includes_server_side_tools(): assert json_result.exit_code == 0 server_side_tools = json.loads(json_result.output)["server_side_tools"] assert [tool["name"] for tool in server_side_tools] == [ + "WebSearch", "CodeInterpreter", "ServerSideTool", ] assert all(tool["server_side"] is True for tool in server_side_tools) - assert server_side_tools[0]["signature"].startswith("(container:") + assert server_side_tools[0]["signature"].startswith("(allowed_domains:") assert server_side_tools[0]["description"].startswith( - "Run Python in an OpenAI-managed container." + "Search the web using OpenAI's hosted search tool." ) diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index a6ea4d065..4bf982d98 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -7,7 +7,7 @@ from pytest_httpx import IteratorStream import llm -from llm.default_plugins.openai_models import CodeInterpreter, Responses +from llm.default_plugins.openai_models import CodeInterpreter, Responses, WebSearch API_KEY = os.environ.get("PYTEST_OPENAI_API_KEY", None) or "badkey" @@ -88,6 +88,158 @@ def test_code_interpreter_prepare_request_is_additive_and_idempotent(): } +@pytest.mark.parametrize( + ("tool", "expected"), + ( + (WebSearch(), {"type": "web_search"}), + ( + WebSearch( + allowed_domains=["openai.com"], + blocked_domains=["example.com"], + user_location={"country": "GB", "city": "London"}, + search_context_size="high", + external_web_access=False, + return_token_budget="unlimited", + search_content_types=["image", "text"], + image_settings={"max_results": 3, "caption": True}, + ), + { + "type": "web_search", + "filters": { + "allowed_domains": ["openai.com"], + "blocked_domains": ["example.com"], + }, + "user_location": { + "type": "approximate", + "country": "GB", + "city": "London", + }, + "search_context_size": "high", + "external_web_access": False, + "return_token_budget": "unlimited", + "search_content_types": ["image", "text"], + "image_settings": {"max_results": 3, "caption": True}, + }, + ), + ), +) +def test_web_search_tool_spec(tool, expected): + assert tool.tool_spec(llm.get_model("gpt-5.6-luna")) == expected + + +@pytest.mark.parametrize( + ("kwargs", "error"), + ( + ({"search_context_size": "huge"}, "search_context_size"), + ({"return_token_budget": "lots"}, "return_token_budget"), + ({"search_content_types": ["video"]}, "search_content_types"), + ({"allowed_domains": ["https://openai.com"]}, "scheme"), + ({"allowed_domains": [f"example{i}.com" for i in range(101)]}, "100"), + ({"user_location": {"type": "exact"}}, "approximate"), + ({"external_web_access": "no"}, "external_web_access"), + ({"image_settings": {"max_results": 0}}, "max_results"), + ({"image_settings": {"caption": "yes"}}, "caption"), + ), +) +def test_web_search_validates_configuration(kwargs, error): + with pytest.raises((TypeError, ValueError), match=error): + WebSearch(**kwargs) + + +def test_web_search_prepare_request_is_additive_and_idempotent(): + tool = WebSearch(include_sources=True, include_results=True) + kwargs = {"include": ["reasoning.encrypted_content"]} + + tool.prepare_request(llm.get_model("gpt-5.6-luna"), kwargs) + tool.prepare_request(llm.get_model("gpt-5.6-luna"), kwargs) + + assert kwargs == { + "include": [ + "reasoning.encrypted_content", + "web_search_call.action.sources", + "web_search_call.results", + ] + } + default_kwargs = {} + WebSearch().prepare_request(llm.get_model("gpt-5.6-luna"), default_kwargs) + assert default_kwargs == {} + + +def test_responses_web_search_request_and_result_capture(httpx_mock): + sources = [ + {"type": "url", "url": "https://openai.com/news/"}, + {"type": "url", "url": "https://example.com/report"}, + ] + results = [ + { + "type": "image_result", + "image_url": "https://example.com/image.jpg", + "source_website_url": "https://example.com/report", + "thumbnail_url": "https://example.com/thumb.jpg", + "caption": "An example image", + } + ] + response_json = _text_response_json(text="A cited answer") + response_json["output"].insert( + 0, + { + "type": "web_search_call", + "id": "ws_123", + "status": "completed", + "action": { + "type": "search", + "query": "OpenAI news", + "queries": ["OpenAI news"], + "sources": sources, + }, + "results": results, + }, + ) + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json=response_json, + headers={"Content-Type": "application/json"}, + ) + + response = llm.get_model("gpt-5.6-luna").prompt( + "Search for OpenAI news", + tools=[ + WebSearch( + allowed_domains=["openai.com"], + include_sources=True, + include_results=True, + ) + ], + stream=False, + key="test", + ) + + assert response.text() == "A cited answer" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["tools"] == [ + { + "type": "web_search", + "filters": {"allowed_domains": ["openai.com"]}, + } + ] + assert request_body["include"] == [ + "reasoning.encrypted_content", + "web_search_call.action.sources", + "web_search_call.results", + ] + message = response.messages()[0] + assert [type(part).__name__ for part in message.parts] == [ + "ToolCallPart", + "ToolResultPart", + "TextPart", + ] + assert message.parts[0].server_executed + assert message.parts[0].arguments["sources"] == sources + assert message.parts[1].server_executed + assert json.loads(message.parts[1].output) == results + + def test_server_side_prepare_request_runs_in_list_order_after_baseline(): class IncludeMarker(CodeInterpreter): def __init__(self, marker): @@ -179,16 +331,20 @@ def test_responses_raw_server_tool_passthrough_on_custom_endpoint(httpx_mock): assert request_body["tools"] == [raw_spec] -def test_code_interpreter_rejected_by_chat_and_chat_fallback(): +@pytest.mark.parametrize("tool", (CodeInterpreter(), WebSearch())) +def test_server_side_tool_rejected_by_chat_and_chat_fallback(tool): from llm.default_plugins.openai_models import Chat chat = Chat("chat-model", supports_tools=True) with pytest.raises(ValueError, match="llm tools -m chat-model"): - chat.prompt("Calculate", tools=[CodeInterpreter()]) + chat.prompt("Use a server-side tool", tools=[tool]) responses_model = llm.get_model("gpt-5.6-luna") response = responses_model.prompt( - "Calculate", tools=[CodeInterpreter()], chat_completions=True, key="test" + "Use a server-side tool", + tools=[tool], + chat_completions=True, + key="test", ) with pytest.raises(ValueError, match="llm tools -m gpt-5.6-luna"): response.text() @@ -224,6 +380,32 @@ async def test_async_responses_code_interpreter_request(httpx_mock): ] +@pytest.mark.asyncio +async def test_async_responses_web_search_request(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json=_text_response_json(), + headers={"Content-Type": "application/json"}, + ) + response = llm.get_async_model("gpt-5.6-luna").prompt( + "Search", + tools=[WebSearch(external_web_access=False, include_sources=True)], + stream=False, + key="test", + ) + + assert await response.text() == "ok" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["tools"] == [ + {"type": "web_search", "external_web_access": False} + ] + assert request_body["include"] == [ + "reasoning.encrypted_content", + "web_search_call.action.sources", + ] + + def _responses_sse(event_type, data): data = {"type": event_type, **data} return f"event: {event_type}\ndata: {json.dumps(data)}\n\n".encode() From df43137d955f2fbb66facd18a03c4eef92d6546a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 19:27:08 -0700 Subject: [PATCH 233/258] Fix server tools for OpenAI endpoint command --- docs/other-models.md | 11 +++++++++ llm/default_plugins/openai_models.py | 7 +++++- tests/test_openai_endpoint.py | 37 ++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/docs/other-models.md b/docs/other-models.md index 6848c2d14..93b7f5169 100644 --- a/docs/other-models.md +++ b/docs/other-models.md @@ -129,6 +129,17 @@ llm openai endpoint https://example.com/v1 \ "What is the capital of France?" ``` +Responses endpoints can define their own server-side tool types. Use `ServerSideTool` to pass an endpoint-specific tool specification through without validation. For example, [OpenRouter's web search server tool](https://openrouter.ai/docs/api/reference/responses/web-search) can be used like this: + +```bash +llm openai endpoint https://openrouter.ai/api/v1 \ + -m openai/gpt-oss-20b:free \ + --key openrouter \ + --responses \ + -T 'ServerSideTool(spec={"type":"openrouter:web_search","parameters":{"engine":"exa","max_results":2,"max_uses":1}})' \ + "Search for the OpenRouter documentation URL" +``` + ### Configure an OpenAI-compatible model The `model_id` is the name LLM will use for the model. The `model_name` is the name which needs to be passed to the API - this might differ from the `model_id`, especially if the `model_id` could potentially clash with other installed models. diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index c6180bb53..67038cb51 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -724,7 +724,12 @@ def endpoint( prompt_kwargs["key"] = key tool_kwargs = _tool_chain_kwargs( - tools, python_tools, tools_debug, tools_approve, chain_limit + tools, + python_tools, + tools_debug, + tools_approve, + chain_limit, + model=model, ) resolved_attachments = [*attachments, *attachment_types] try: diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py index cf68934a2..900e9defe 100644 --- a/tests/test_openai_endpoint.py +++ b/tests/test_openai_endpoint.py @@ -889,6 +889,43 @@ def test_endpoint_responses_api_tools(httpx_mock, user_path): ] +def test_endpoint_responses_api_raw_server_side_tool(httpx_mock, user_path): + base_url = "https://raw-tools.example.test/v1" + httpx_mock.add_response( + method="POST", + url=f"{base_url}/responses", + json=_responses_payload("Search complete"), + headers={"Content-Type": "application/json"}, + ) + tool_spec = { + "type": "openrouter:web_search", + "parameters": {"engine": "exa", "max_results": 2, "max_uses": 1}, + } + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Search for the latest news", + "-m", + "test-model", + "--responses", + "--no-stream", + "-T", + f"ServerSideTool(spec={json.dumps(tool_spec)})", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "Search complete\n" + assert not (user_path / "logs.db").exists() + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["tools"] == [tool_spec] + + def test_endpoint_reads_one_off_prompt_from_stdin(httpx_mock, user_path): base_url = "https://stdin.example.test/v1" _add_chat_response(httpx_mock, base_url, "From stdin") From 56207127821424554dd6a1ad40b0102c5c6fd343 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 19:51:14 -0700 Subject: [PATCH 234/258] Persist server tool configuration across continuations --- docs/usage.md | 2 +- llm/cli.py | 24 ++++--- llm/logs.py | 16 +++-- llm/models.py | 7 +- tests/test_cli_openai_models.py | 123 ++++++++++++++++++++++++++++++++ tests/test_tools.py | 16 +++++ 6 files changed, 171 insertions(+), 17 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index d8f88b64b..6f0f3c04e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -179,7 +179,7 @@ llm tools -m gpt-5.6-luna These are displayed in a separate `Server-side tools` section with their constructor signatures and documentation. Add `--json` to return them in a `server_side_tools` array whose entries have `"server_side": true`. -If you run a prompt that uses tools from plugins (as opposed to tools provided using the `--functions` option) continuing that conversation using `llm -c` will reuse the tools from the first prompt. Running `llm chat -c` will start a chat that continues using those same tools. For example: +If you run a prompt that uses tools from plugins or model-specific server-side tools, continuing that conversation using `llm -c` will reuse the tools from the first prompt. Configured constructor arguments such as `CodeInterpreter(memory_limit="4g")` are retained. Running `llm chat -c` will start a chat that continues using those same tools. For example: ``` llm -T simple_eval "12345 * 12345" --td diff --git a/llm/cli.py b/llm/cli.py index c07a87e66..67a89d3f1 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -1445,28 +1445,36 @@ def load_conversation( except KeyError: pass - # Plugin tools recorded against the first turn, for the same - # reuse-on-continue behaviour the rebuilt responses provide. Tools - # that came from a toolbox are collapsed into a single spec string - # like Datasette({"url": "..."}) - the same format -T accepts - so - # the instance can be reconstructed with its configuration. + # Plugin and server-side tools recorded against the first turn, for + # the same reuse-on-continue behaviour the rebuilt responses provide. + # Configured instances are collapsed into a single spec string like + # Datasette({"url": "..."}) - the same format -T accepts - so the + # instance can be reconstructed with its configuration. loaded_tools = [] seen_instance_ids = set() + supported_server_side_tool_names = { + tool_class.__name__ + for tool_class in conversation.model.supported_server_side_tools + } for tool_row in db.query( """ - select tools.name, turn_tools.instance_id, + select tools.name, tools.plugin, turn_tools.instance_id, tool_instances.name as instance_name, tool_instances.arguments as instance_arguments from tools join turn_tools on turn_tools.tool_id = tools.id left join tool_instances on tool_instances.id = turn_tools.instance_id - where tools.plugin is not null - and turn_tools.turn_id = ( + where turn_tools.turn_id = ( select id from turns where thread_id = ? order by id limit 1 ) """, [conversation_id], ): + if ( + tool_row["plugin"] is None + and tool_row["instance_name"] not in supported_server_side_tool_names + ): + continue if tool_row["instance_id"] is None: loaded_tools.append(tool_row["name"]) elif tool_row["instance_id"] not in seen_instance_ids: diff --git a/llm/logs.py b/llm/logs.py index 744efb9f0..fc4839265 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -24,7 +24,7 @@ from condense_json import UncondenseError, condense_json, uncondense_json from .migrations import migrate -from .models import Attachment, _conversation_name +from .models import Attachment, ServerSideTool, _conversation_name from .parts import ( AttachmentPart, Message, @@ -528,10 +528,16 @@ def _log_in_transaction(self, response, thread_id: str | None) -> str: replace=True, ) for tool in response.prompt.tools: - # A toolbox-derived tool's implementation is a method bound - # to the configured instance - record which one, as a + # Server-side tools are configured instances themselves. A + # toolbox-derived tool instead has an implementation method + # bound to its configured instance. Record either kind as a # reference into the shared tool_instances table. - instance = getattr(tool.implementation, "__self__", None) + if isinstance(tool, ServerSideTool): + instance = tool + instance_name = tool.__class__.__name__ + else: + instance = getattr(tool.implementation, "__self__", None) + instance_name = tool.name.split("_")[0] config = getattr(instance, "_config", None) self.db["turn_tools"].insert( { @@ -540,7 +546,7 @@ def _log_in_transaction(self, response, thread_id: str | None) -> str: "instance_id": ( ensure_tool_instance( self.db, - tool.name.split("_")[0], + instance_name, tool.plugin, json.dumps(config), ) diff --git a/llm/models.py b/llm/models.py index 1c7657f9d..d7f310448 100644 --- a/llm/models.py +++ b/llm/models.py @@ -734,9 +734,10 @@ class _BaseConversation: # exact message list, so reasoning signatures and provider metadata # survive being reloaded. loaded_messages: list[Any] | None = None - # Plugin tool names and toolbox specs (e.g. 'Datasette({"url": ...})') - # recorded against this conversation's first turn in storage. Read - # when the conversation was loaded from the message store, where + # Plugin and server-side tool names and configured specs (e.g. + # 'Datasette({"url": ...})' or 'CodeInterpreter({"memory_limit": + # "4g"})') recorded against this conversation's first turn in storage. + # Read when the conversation was loaded from the message store, where # there are no rebuilt responses to copy prompt.tools from. loaded_tools: list[str] | None = None diff --git a/tests/test_cli_openai_models.py b/tests/test_cli_openai_models.py index b550d32a1..af6ee53e6 100644 --- a/tests/test_cli_openai_models.py +++ b/tests/test_cli_openai_models.py @@ -271,6 +271,129 @@ def test_code_interpreter_cli_tool_is_resolved_from_model(httpx_mock): assert "code_interpreter_call.outputs" in request_body["include"] +def test_code_interpreter_cli_tool_is_reused_on_continue(httpx_mock, user_path): + def response_payload(response_id, text): + return { + "id": response_id, + "object": "response", + "created_at": 1, + "model": "gpt-5.6-luna", + "output": [ + { + "type": "message", + "id": f"msg_{response_id}", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": text, + "annotations": [], + } + ], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + "status": "completed", + } + + first_payload = response_payload("resp_code_interpreter_first", "Calculated") + first_payload["output"] = [ + { + "type": "message", + "id": "msg_before_code", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Running Python", + "annotations": [], + } + ], + }, + { + "type": "code_interpreter_call", + "id": "ci_continue", + "status": "completed", + "container_id": "cntr_continue", + "code": "print(6 * 7)", + "outputs": [{"type": "logs", "logs": "42\n"}], + }, + { + "type": "message", + "id": "msg_after_code", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Calculated", + "annotations": [], + } + ], + }, + ] + for payload in ( + first_payload, + response_payload("resp_code_interpreter_second", "Continued"), + ): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json=payload, + headers={"Content-Type": "application/json"}, + ) + + runner = CliRunner() + first = runner.invoke( + cli, + [ + "-m", + "gpt-5.6-luna", + "-T", + 'CodeInterpreter(memory_limit="4g")', + "--no-stream", + "--key", + "x", + "Run this calculation", + ], + catch_exceptions=False, + ) + second = runner.invoke( + cli, + ["Continue", "-c", "--no-stream", "--key", "x"], + catch_exceptions=False, + ) + + assert first.exit_code == 0 + assert second.exit_code == 0 + assert second.output == "Continued\n" + request_bodies = [ + json.loads(request.content) for request in httpx_mock.get_requests() + ] + expected_tool = { + "type": "code_interpreter", + "container": {"type": "auto", "memory_limit": "4g"}, + } + assert request_bodies[0]["tools"] == [expected_tool] + assert request_bodies[1]["tools"] == [expected_tool] + assert request_bodies[1]["input"] == [ + {"role": "user", "content": "Run this calculation"}, + {"role": "assistant", "content": "Running Python"}, + {"role": "assistant", "content": "Calculated"}, + {"role": "user", "content": "Continue"}, + ] + + db = sqlite_utils.Database(str(user_path / "logs.db")) + instance = next(iter(db["tool_instances"].rows)) + assert instance["name"] == "CodeInterpreter" + assert json.loads(instance["arguments"])["memory_limit"] == "4g" + assert {row["instance_id"] for row in db["turn_tools"].rows} == { + instance["id"] + } + + def test_web_search_cli_tool_is_resolved_from_model(httpx_mock): httpx_mock.add_response( method="POST", diff --git a/tests/test_tools.py b/tests/test_tools.py index 1597a06e2..21f18fe20 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -134,6 +134,22 @@ def supported_server_side_tools(self): ConditionalModel(False).prompt("hello", tools=[DemoServerSideTool()]) +def test_server_side_tool_configuration_is_logged(): + db = sqlite_utils.Database(memory=True) + migrate(db) + response = ServerToolsOnlyModel().prompt( + "hello", tools=[DemoServerSideTool("configured")] + ) + response.text() + response.log_to_db(db) + + instance = next(iter(db["tool_instances"].rows)) + assert instance["name"] == "DemoServerSideTool" + assert instance["plugin"] is None + assert json.loads(instance["arguments"]) == {"value": "configured"} + assert next(iter(db["turn_tools"].rows))["instance_id"] == instance["id"] + + def test_function_tools_still_require_supports_tools(): def local_tool(): return "local" From f18c88ad20089c81c405932ef1a981bf67f77641 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 21:27:25 -0700 Subject: [PATCH 235/258] Add JSON output for model listings --- docs/help.md | 1 + docs/usage.md | 7 +++++++ llm/cli.py | 35 ++++++++++++++++++++++++++++++++++- tests/test_llm.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 1 deletion(-) diff --git a/docs/help.md b/docs/help.md index c34422de0..96f062794 100644 --- a/docs/help.md +++ b/docs/help.md @@ -402,6 +402,7 @@ Options: --async List async models --schemas List models that support schemas --tools List models that support tools + --json Output as JSON -q, --query TEXT Search for models matching these strings -m, --model TEXT Specific model IDs -h, --help Show this message and exit. diff --git a/docs/usage.md b/docs/usage.md index 6f0f3c04e..93352b316 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -591,6 +591,13 @@ Use one or more `-m` options to indicate specific models, either by their model ```bash llm models -m gpt-5.6-luna -m claude-opus-4.8 ``` + +Add `--json` to return an array of model records with aliases, capability flags, attachment types and `server_side_tools`. Combine it with `-m` to inspect one or more specific models; adding `--options` includes each model's option schemas: + +```bash +llm models --json -m gpt-5.6-luna +``` + Add `--options` to also see documentation for the options supported by each model: ```bash llm models --options diff --git a/llm/cli.py b/llm/cli.py index 67a89d3f1..f8a2d3a87 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -2405,6 +2405,7 @@ def render_model_with_options(model_id, *, async_=False): @click.option("async_", "--async", is_flag=True, help="List async models") @click.option("--schemas", is_flag=True, help="List models that support schemas") @click.option("--tools", is_flag=True, help="List models that support tools") +@click.option("json_", "--json", is_flag=True, help="Output as JSON") @click.option( "-q", "--query", @@ -2412,9 +2413,10 @@ def render_model_with_options(model_id, *, async_=False): help="Search for models matching these strings", ) @click.option("model_ids", "-m", "--model", help="Specific model IDs", multiple=True) -def models_list(options, async_, schemas, tools, query, model_ids): +def models_list(options, async_, schemas, tools, json_, query, model_ids): "List available models" models_that_have_shown_options = set() + json_models = [] for model_with_aliases in get_models_with_aliases(): if async_ and not model_with_aliases.async_model: continue @@ -2427,6 +2429,34 @@ def models_list(options, async_, schemas, tools, query, model_ids): continue if tools and not model_with_aliases.model.supports_tools: continue + if json_: + model = ( + model_with_aliases.async_model + if async_ + else model_with_aliases.model + ) + model_json = { + "model_id": model.model_id, + "aliases": model_with_aliases.aliases, + "can_stream": model.can_stream, + "supports_schema": model.supports_schema, + "supports_tools": model.supports_tools, + "supports_async": model_with_aliases.async_model is not None, + "attachment_types": sorted(model.attachment_types), + "server_side_tools": [ + { + "name": tool_class.__name__, + "plugin": getattr(tool_class, "plugin", None), + } + for tool_class in model.supported_server_side_tools + ], + } + if options: + model_json["options"] = model.Options.model_json_schema()[ + "properties" + ] + json_models.append(model_json) + continue click.echo( render_model_with_aliases( model_with_aliases, @@ -2435,6 +2465,9 @@ def models_list(options, async_, schemas, tools, query, model_ids): models_that_have_shown_options=models_that_have_shown_options, ) ) + if json_: + click.echo(json.dumps(json_models, indent=2)) + return if not query and not options and not schemas and not model_ids: click.echo(f"Default: {get_default_model()}") diff --git a/tests/test_llm.py b/tests/test_llm.py index 0a201e7ef..14407165b 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -517,6 +517,52 @@ def test_llm_models_async(user_path): assert "AsyncMockModel (async): mock" in result.output +def test_llm_models_json_includes_server_side_tools(): + runner = CliRunner() + result = runner.invoke( + cli, + [ + "models", + "--json", + "-m", + "gpt-5.6-luna", + "-m", + "chatgpt", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + models = {model["model_id"]: model for model in json.loads(result.output)} + assert set(models) == {"gpt-5.6-luna", "gpt-3.5-turbo"} + assert models["gpt-5.6-luna"]["server_side_tools"] == [ + {"name": "WebSearch", "plugin": None}, + {"name": "CodeInterpreter", "plugin": None}, + {"name": "ServerSideTool", "plugin": None}, + ] + assert models["gpt-3.5-turbo"]["server_side_tools"] == [] + assert models["gpt-5.6-luna"]["supports_schema"] is True + assert models["gpt-5.6-luna"]["supports_tools"] is True + assert models["gpt-5.6-luna"]["can_stream"] is True + assert models["gpt-5.6-luna"]["supports_async"] is True + assert "application/pdf" in models["gpt-5.6-luna"]["attachment_types"] + + +def test_llm_models_json_options_and_alias_filter(): + result = CliRunner().invoke( + cli, + ["models", "--json", "--options", "-m", "4.1"], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + models = json.loads(result.output) + assert len(models) == 1 + assert models[0]["model_id"] == "gpt-4.1" + assert "4.1" in models[0]["aliases"] + assert "temperature" in models[0]["options"] + + @pytest.mark.parametrize( "args,expected_model_ids,unexpected_model_ids", ( From 8af0b9319b0a71b72d32a9d56a6c2cd275573850 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 21:38:12 -0700 Subject: [PATCH 236/258] Render schema-less tools in expanded logs --- llm/cli.py | 2 +- tests/test_tools.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/llm/cli.py b/llm/cli.py index f8a2d3a87..87d2868a0 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -2180,7 +2180,7 @@ def echo_tool(tool, indent=""): textwrap.indent( (tool["description"] or "").rstrip(), " " ), - json.dumps(tool["input_schema"]["properties"]), + json.dumps(tool["input_schema"].get("properties", {})), ) click.echo(textwrap.indent(block, indent)) diff --git a/tests/test_tools.py b/tests/test_tools.py index 21f18fe20..b09e7050d 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -150,6 +150,22 @@ def test_server_side_tool_configuration_is_logged(): assert next(iter(db["turn_tools"].rows))["instance_id"] == instance["id"] +def test_logs_expanded_server_side_tool(user_path): + db = sqlite_utils.Database(str(user_path / "logs.db")) + migrate(db) + response = ServerToolsOnlyModel().prompt( + "hello", tools=[DemoServerSideTool("configured")] + ) + response.text() + response.log_to_db(db) + + result = CliRunner().invoke(cli.cli, ["logs", "-cue"], catch_exceptions=False) + + assert result.exit_code == 0 + assert '- `DemoServerSideTool({"value": "configured"})`:' in result.output + assert "Arguments: `{}`" in result.output + + def test_function_tools_still_require_supports_tools(): def local_tool(): return "local" From ab19d6a59de4c085ce7d94014b911c373b44d80d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 4 Aug 2026 04:41:06 +0000 Subject: [PATCH 237/258] Ran cog --- README.md | 3 +++ docs/help.md | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7392c8a2a..247086a69 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,8 @@ For everything else, see [the llm tag](https://simonwillison.net/tags/llm/) on m * [Configuration](https://llm.datasette.io/en/stable/openai-models.html#configuration) * [OpenAI language models](https://llm.datasette.io/en/stable/openai-models.html#openai-language-models) * [Model features](https://llm.datasette.io/en/stable/openai-models.html#model-features) + * [Web Search](https://llm.datasette.io/en/stable/openai-models.html#web-search) + * [Code Interpreter](https://llm.datasette.io/en/stable/openai-models.html#code-interpreter) * [Fast mode and service tiers](https://llm.datasette.io/en/stable/openai-models.html#fast-mode-and-service-tiers) * [OpenAI embedding models](https://llm.datasette.io/en/stable/openai-models.html#openai-embedding-models) * [OpenAI completion models](https://llm.datasette.io/en/stable/openai-models.html#openai-completion-models) @@ -305,6 +307,7 @@ For everything else, see [the llm tag](https://simonwillison.net/tags/llm/) on m * [Async models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#async-models) * [Supporting schemas](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-schemas) * [Supporting tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-tools) + * [Supporting server-side tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-server-side-tools) * [Attachments for multi-modal models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#attachments-for-multi-modal-models) * [Structured messages and streaming events](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#structured-messages-and-streaming-events) * [Condensing logged payloads with json_replacements](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#condensing-logged-payloads-with-json-replacements) diff --git a/docs/help.md b/docs/help.md index 96f062794..391900318 100644 --- a/docs/help.md +++ b/docs/help.md @@ -640,7 +640,7 @@ Options: -h, --help Show this message and exit. Commands: - list* List available tools that have been provided by plugins + list* List available tools, optionally including tools supported by a model ``` (help-tools-list)= @@ -648,10 +648,11 @@ Commands: ``` Usage: llm tools list [OPTIONS] [TOOL_DEFS]... - List available tools that have been provided by plugins + List available tools, optionally including tools supported by a model Options: --json Output as JSON + -m, --model TEXT List tools supported by this model --functions TEXT Python code block or file path defining functions to register as tools -h, --help Show this message and exit. From 0396898c3fa1fdf810d4c1696decff5844b444ca Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 21:51:30 -0700 Subject: [PATCH 238/258] Format server-side tool changes --- llm/cli.py | 12 +++--------- llm/default_plugins/openai_models.py | 7 ++----- llm/models.py | 8 +++++--- tests/test_cli_openai_models.py | 8 ++------ tests/test_openai_responses.py | 4 +--- tests/test_tools.py | 14 ++++++++------ 6 files changed, 21 insertions(+), 32 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index 87d2868a0..22c9b58c1 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -2431,9 +2431,7 @@ def models_list(options, async_, schemas, tools, json_, query, model_ids): continue if json_: model = ( - model_with_aliases.async_model - if async_ - else model_with_aliases.model + model_with_aliases.async_model if async_ else model_with_aliases.model ) model_json = { "model_id": model.model_id, @@ -2452,9 +2450,7 @@ def models_list(options, async_, schemas, tools, json_, query, model_ids): ], } if options: - model_json["options"] = model.Options.model_json_schema()[ - "properties" - ] + model_json["options"] = model.Options.model_json_schema()["properties"] json_models.append(model_json) continue click.echo( @@ -2848,9 +2844,7 @@ def introspect_tools(toolbox): output = {"tools": output_tools, "toolboxes": output_toolboxes} if model is not None: output["server_side_tools"] = server_side_tools - click.echo( - json.dumps(output, indent=2) - ) + click.echo(json.dumps(output, indent=2)) else: for tool in tool_objects: sig = "()" diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 67038cb51..65b73ebb5 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1777,9 +1777,7 @@ def __init__( if container is not None and not isinstance(container, str): raise TypeError("container must be a string container ID") if memory_limit is not None and memory_limit not in self._memory_limits: - raise ValueError( - "memory_limit must be one of: 1g, 4g, 16g or 64g" - ) + raise ValueError("memory_limit must be one of: 1g, 4g, 16g or 64g") if container is not None and (memory_limit is not None or file_ids is not None): raise ValueError( "container cannot be combined with memory_limit or file_ids" @@ -2094,8 +2092,7 @@ def _finalize_responses_kwargs(self, prompt, stream, instructions=None): kwargs["instructions"] = instructions kwargs["store"] = False if self._reasoning and ( - self._reasoning_summary - or getattr(prompt.options, "reasoning_effort", None) + self._reasoning_summary or getattr(prompt.options, "reasoning_effort", None) ): include = kwargs.setdefault("include", []) if "reasoning.encrypted_content" not in include: diff --git a/llm/models.py b/llm/models.py index d7f310448..6e616b904 100644 --- a/llm/models.py +++ b/llm/models.py @@ -627,9 +627,11 @@ def _partition_tools( # instances. Without this exact-type exception it would also # accidentally claim every provider-specific subclass. claimed = any( - type(tool) is candidate - if candidate is ServerSideTool - else isinstance(tool, candidate) + ( + type(tool) is candidate + if candidate is ServerSideTool + else isinstance(tool, candidate) + ) for candidate in declared ) if not claimed: diff --git a/tests/test_cli_openai_models.py b/tests/test_cli_openai_models.py index af6ee53e6..27ebe5fd8 100644 --- a/tests/test_cli_openai_models.py +++ b/tests/test_cli_openai_models.py @@ -389,9 +389,7 @@ def response_payload(response_id, text): instance = next(iter(db["tool_instances"].rows)) assert instance["name"] == "CodeInterpreter" assert json.loads(instance["arguments"])["memory_limit"] == "4g" - assert {row["instance_id"] for row in db["turn_tools"].rows} == { - instance["id"] - } + assert {row["instance_id"] for row in db["turn_tools"].rows} == {instance["id"]} def test_web_search_cli_tool_is_resolved_from_model(httpx_mock): @@ -469,9 +467,7 @@ def test_tools_list_for_model_includes_server_side_tools(): assert "Run Python in an OpenAI-managed container." in result.output assert "ServerSideTool(spec: dict | None = None)" in result.output - json_result = runner.invoke( - cli, ["tools", "-m", "gpt-5.6-luna", "--json"] - ) + json_result = runner.invoke(cli, ["tools", "-m", "gpt-5.6-luna", "--json"]) assert json_result.exit_code == 0 server_side_tools = json.loads(json_result.output)["server_side_tools"] assert [tool["name"] for tool in server_side_tools] == [ diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index 4bf982d98..ddba5ed40 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -24,9 +24,7 @@ def _text_response_json(model="gpt-5.6-luna", text="ok"): "id": "msg_server_tool", "role": "assistant", "status": "completed", - "content": [ - {"type": "output_text", "text": text, "annotations": []} - ], + "content": [{"type": "output_text", "text": text, "annotations": []}], } ], "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, diff --git a/tests/test_tools.py b/tests/test_tools.py index b09e7050d..97dc3d22b 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -111,9 +111,10 @@ def test_unsupported_server_side_tool_fails_before_execution(mock_model): def test_declaring_raw_escape_hatch_does_not_claim_every_subclass(): model = RawServerToolOnlyModel() - assert model.prompt( - "hello", tools=[llm.ServerSideTool({"type": "custom"})] - ).text() == "done" + assert ( + model.prompt("hello", tools=[llm.ServerSideTool({"type": "custom"})]).text() + == "done" + ) with pytest.raises(ValueError, match="does not support server-side tool"): model.prompt("hello", tools=[DemoServerSideTool()]) @@ -127,9 +128,10 @@ def __init__(self, enabled): def supported_server_side_tools(self): return (DemoServerSideTool,) if self.enabled else () - assert ConditionalModel(True).prompt( - "hello", tools=[DemoServerSideTool()] - ).text() == "done" + assert ( + ConditionalModel(True).prompt("hello", tools=[DemoServerSideTool()]).text() + == "done" + ) with pytest.raises(ValueError, match="does not support server-side tool"): ConditionalModel(False).prompt("hello", tools=[DemoServerSideTool()]) From b027a3c4997575327ca7d5eeb6700e4357cbd787 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 22:01:08 -0700 Subject: [PATCH 239/258] Fix tool instance type annotation --- llm/logs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llm/logs.py b/llm/logs.py index fc4839265..159ad2996 100644 --- a/llm/logs.py +++ b/llm/logs.py @@ -532,6 +532,7 @@ def _log_in_transaction(self, response, thread_id: str | None) -> str: # toolbox-derived tool instead has an implementation method # bound to its configured instance. Record either kind as a # reference into the shared tool_instances table. + instance: Any | None if isinstance(tool, ServerSideTool): instance = tool instance_name = tool.__class__.__name__ From a66dca7911da6795a5c742caf09be8680d09e70b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 22:01:08 -0700 Subject: [PATCH 240/258] Handle custom Responses output without warnings --- docs/other-models.md | 3 +++ llm/default_plugins/openai_models.py | 8 ++++---- tests/test_openai_endpoint.py | 19 +++++++++++++++++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/docs/other-models.md b/docs/other-models.md index 93b7f5169..a2a8dd695 100644 --- a/docs/other-models.md +++ b/docs/other-models.md @@ -136,10 +136,13 @@ llm openai endpoint https://openrouter.ai/api/v1 \ -m openai/gpt-oss-20b:free \ --key openrouter \ --responses \ + -R \ -T 'ServerSideTool(spec={"type":"openrouter:web_search","parameters":{"engine":"exa","max_results":2,"max_uses":1}})' \ "Search for the OpenRouter documentation URL" ``` +`-R` hides the reasoning text returned by this model so the command displays just its final answer. + ### Configure an OpenAI-compatible model The `model_id` is the name LLM will use for the model. The `model_name` is the name which needs to be passed to the API - this might differ from the `model_id`, especially if the `model_id` could potentially clash with other installed models. diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 65b73ebb5..6854afbf9 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -2558,7 +2558,7 @@ def execute( else: yield from self._server_tool_events(item, message_index) elif etype == "response.completed": - final_response_dict = event.response.model_dump() + final_response_dict = event.response.model_dump(warnings=False) if final_response_dict.get("usage"): usage = final_response_dict["usage"] if final_response_dict is not None: @@ -2573,7 +2573,7 @@ def execute( stream=False, **kwargs, ) - dumped = completion.model_dump() + dumped = completion.model_dump(warnings=False) response.response_json = remove_dict_none_values(dumped) usage = dumped.get("usage") events, had_reasoning = self._non_streaming_output_events( @@ -2799,7 +2799,7 @@ async def execute( ): yield server_event elif etype == "response.completed": - final_response_dict = event.response.model_dump() + final_response_dict = event.response.model_dump(warnings=False) if final_response_dict.get("usage"): usage = final_response_dict["usage"] if final_response_dict is not None: @@ -2815,7 +2815,7 @@ async def execute( stream=False, **kwargs, ) - dumped = completion.model_dump() + dumped = completion.model_dump(warnings=False) response.response_json = remove_dict_none_values(dumped) usage = dumped.get("usage") events, had_reasoning = self._non_streaming_output_events( diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py index 900e9defe..d6844d9b0 100644 --- a/tests/test_openai_endpoint.py +++ b/tests/test_openai_endpoint.py @@ -889,12 +889,26 @@ def test_endpoint_responses_api_tools(httpx_mock, user_path): ] -def test_endpoint_responses_api_raw_server_side_tool(httpx_mock, user_path): +def test_endpoint_responses_api_raw_server_side_tool(httpx_mock, user_path, recwarn): base_url = "https://raw-tools.example.test/v1" + response_payload = _responses_payload("Search complete") + response_payload["output"].insert( + 0, + { + "id": "st_test", + "type": "openrouter:web_search", + "status": "completed", + "action": { + "type": "search", + "query": "latest news", + "sources": [{"type": "url", "url": "https://example.test/news"}], + }, + }, + ) httpx_mock.add_response( method="POST", url=f"{base_url}/responses", - json=_responses_payload("Search complete"), + json=response_payload, headers={"Content-Type": "application/json"}, ) tool_spec = { @@ -924,6 +938,7 @@ def test_endpoint_responses_api_raw_server_side_tool(httpx_mock, user_path): assert not (user_path / "logs.db").exists() request_body = json.loads(httpx_mock.get_requests()[0].content) assert request_body["tools"] == [tool_spec] + assert not recwarn.list def test_endpoint_reads_one_off_prompt_from_stdin(httpx_mock, user_path): From c97c11a72cc964a9da211ab768d53b8cdb0f52f5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 22:10:50 -0700 Subject: [PATCH 241/258] Removed confusing sentence --- docs/plugins/advanced-model-plugins.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index d2d5a94eb..9e6564ceb 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -203,7 +203,7 @@ Providers with an OpenAI-compatible tools array can optionally support raw speci llm.ServerSideTool({"type": "browser_search"}) ``` -Declaring the base class does not claim subclasses belonging to other providers. Unsupported combinations raise an error instead of silently dropping or serializing the tool as a function tool. Server-side tool calls returned by the provider should use the `server_executed=True` events described in {ref}`structured-messages-streaming`. +Server-side tool calls returned by the provider should use the `server_executed=True` events described in {ref}`structured-messages-streaming`. ```{eval-rst} .. autoclass:: llm.ServerSideTool From 2e31ab0663e86ed04542466a47a7cbf059c5bcd5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 3 Aug 2026 22:19:29 -0700 Subject: [PATCH 242/258] Document server-side tools in changelog --- docs/changelog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 30a2439a6..5f813f2e1 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -35,6 +35,9 @@ These APIs were introduced in {ref}`0.32a0 `, which has a more detaile - Async sibling calls finish before a pause or callback failure is propagated, avoiding orphaned work. Missing async tools now produce the same error results as synchronous execution. - Conversations that use configured toolboxes can be continued with `llm -c` or `llm chat -c` without repeating the toolbox configuration. - `llm tools` now shows constructor signatures and docstrings for dynamic toolboxes. Passing a toolbox specification instantiates it and lists its runtime-generated tools, while `llm tools --json` identifies dynamic toolboxes with a `"dynamic"` boolean. [#1580](https://github.com/simonw/llm/issues/1580) +- Models can now declare the server-side tools they support using the instance-level `supported_server_side_tools` property and the new `llm.ServerSideTool` base class. OpenAI Responses API models provide `WebSearch` and `CodeInterpreter`, available from the CLI using `-T WebSearch` or `-T 'CodeInterpreter(memory_limit="4g")'`. Server-executed calls and results are captured as structured message parts, and `llm -c` restores configured server-side tools for continued conversations. [#1592](https://github.com/simonw/llm/issues/1592), [#1593](https://github.com/simonw/llm/pull/1593) +- `llm tools -m MODEL` lists the server-side tools supported by that model. `llm models --json` returns model aliases, capability flags, attachment types and server-side tools, with option schemas included when combined with `--options`. +- OpenAI-compatible Responses endpoints can use provider-specific server-side tools with `ServerSideTool(spec={...})`, including OpenRouter's web search implementation. See {ref}`0.32a3 ` for more detail on pausing, resuming and inspecting tool calls. From ad0909e423ed2eb76c9bf9e8c20c5c5e263090fb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Aug 2026 10:01:42 -0700 Subject: [PATCH 243/258] Edits to changelog for 0.32, refs #1590 --- docs/changelog.md | 34 ++++++++++++++++------------------ docs/python-api.md | 2 ++ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 5f813f2e1..36f0d4e0e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -9,48 +9,46 @@ LLM 0.32 is a major, backwards-compatible update to the way prompts, responses, Prompt inputs and model outputs are now represented as lists of `Message` objects, each containing typed `Part` objects for text, reasoning, tool calls, tool results and attachments. -- New `llm.Message` value type and constructor helpers `llm.user()`, `llm.assistant()`, `llm.system()` and `llm.tool_message()`. -- New `messages=` keyword argument on the prompt, conversation and chain APIs, including their asynchronous equivalents. Existing `prompt=`, `system=`, `attachments=` and `tool_results=` arguments continue to work and are converted into the same structured representation. -- New `response.stream_events()` and `response.astream_events()` methods expose mixed streams of text, reasoning, tool calls and tool results. Iterating over a response directly continues to yield text strings. +- New {ref}`messages= keyword argument ` on the prompt, conversation and chain APIs, including their asynchronous equivalents. For example, `model.prompt(messages=[llm.user("Hello"), llm.assistant("Hi!"), llm.user("What can you do?")])`. Existing `prompt=`, `system=`, `attachments=` and `tool_results=` arguments continue to work and are converted into the same structured representation. +- New {ref}`structured streaming methods ` `response.stream_events()` and `response.astream_events()` expose mixed streams of text, reasoning, tool calls and tool results. Iterating over a response directly continues to yield text strings. - `response.messages()` returns the assembled structured output. `response.reply()` continues from any response and can automatically execute pending tool calls before the next turn. - `response.to_dict()` and `Response.from_dict()` provide JSON-safe persistence of complete turns, including reasoning and provider metadata, with matching `TypedDict` definitions in the new `llm.serialization` module. - `response.prompt.messages` is now the canonical record of exactly what was sent to the model, including the full preceding conversation chain. -These APIs were introduced in {ref}`0.32a0 `, which has a more detailed inventory and links to the expanded {ref}`Advanced model plugins ` documentation. +These APIs were introduced in {ref}`0.32a0 `. They are described in the {ref}`Advanced model plugins ` documentation. ### OpenAI Responses API and reasoning - Most reasoning-capable OpenAI models now use the `/v1/responses` endpoint by default, enabling interleaved reasoning across tool calls. The existing Chat Completions classes remain available, and `-o chat_completions 1` selects that older path for an individual prompt. See {ref}`0.32a2 ` for the full list of affected models. -- Visible reasoning summaries are streamed to standard error by `llm prompt` and `llm chat`. Use `-R/--hide-reasoning` or the new `hide_reasoning=True` Python argument to hide them. Encrypted reasoning metadata is preserved for subsequent turns. +- OpenAI Responses API models now provide {ref}`WebSearch ` and {ref}`CodeInterpreter ` server-side tools, available from the CLI using `-T WebSearch` or `-T 'CodeInterpreter(memory_limit="4g")'`. +- {ref}`Visible reasoning summaries ` are streamed to standard error by `llm prompt` and `llm chat`. Use `-R/--hide-reasoning` or the new `hide_reasoning=True` Python argument to hide them. Encrypted reasoning metadata is preserved for subsequent turns. - The default model for users who have not selected one is now [GPT-5.6 Luna](https://developers.openai.com/api/docs/models/gpt-5.6-luna), replacing GPT-4o mini. New built-in models include `gpt-5.6-sol`, `gpt-5.6-terra` and `gpt-5.6-luna`; models that are no longer available from OpenAI have been removed. See {ref}`0.32rc1 ` and {ref}`0.32rc2 ` for details. - New {ref}`llm openai endpoint ` command runs prompts and chats, or lists models, against an arbitrary OpenAI-compatible endpoint without configuring it first. These calls are not logged. - OpenAI models now support a `service_tier` option. Use `-o service_tier fast` for faster responses at a higher price, or `-o service_tier flex` for slower, cheaper processing on supported models. See {ref}`Fast mode and service tiers `. [#1585](https://github.com/simonw/llm/pull/1585) -- New `llm -m model --options` flag lists the options supported by a model. The Python prompt APIs now accept an explicit `options=` dictionary as well as the previous keyword-argument form. +- New `llm -m model --options` flag lists the options supported by a model. The Python prompt APIs now accept an explicit {ref}`options= dictionary ` as well as the previous keyword-argument form. ### More controllable tool loops -- Every tool call now has a unique `tool_call_id`, synthesized when the provider does not supply one. Tool implementations can accept an `llm_tool_call` parameter to inspect the current call and its ID. -- Tools can raise `llm.PauseChain` to pause execution for human approval or another external event. Chains can later resume from a message history ending in unresolved tool calls, without repeating calls that already have results. -- `response.execute_tool_call()` executes one call, while `response.execute_tool_calls(tool_calls_list=...)` can execute an explicit list. Both are awaitable on asynchronous responses. -- Async sibling calls finish before a pause or callback failure is propagated, avoiding orphaned work. Missing async tools now produce the same error results as synchronous execution. -- Conversations that use configured toolboxes can be continued with `llm -c` or `llm chat -c` without repeating the toolbox configuration. -- `llm tools` now shows constructor signatures and docstrings for dynamic toolboxes. Passing a toolbox specification instantiates it and lists its runtime-generated tools, while `llm tools --json` identifies dynamic toolboxes with a `"dynamic"` boolean. [#1580](https://github.com/simonw/llm/issues/1580) -- Models can now declare the server-side tools they support using the instance-level `supported_server_side_tools` property and the new `llm.ServerSideTool` base class. OpenAI Responses API models provide `WebSearch` and `CodeInterpreter`, available from the CLI using `-T WebSearch` or `-T 'CodeInterpreter(memory_limit="4g")'`. Server-executed calls and results are captured as structured message parts, and `llm -c` restores configured server-side tools for continued conversations. [#1592](https://github.com/simonw/llm/issues/1592), [#1593](https://github.com/simonw/llm/pull/1593) -- `llm tools -m MODEL` lists the server-side tools supported by that model. `llm models --json` returns model aliases, capability flags, attachment types and server-side tools, with option schemas included when combined with `--options`. -- OpenAI-compatible Responses endpoints can use provider-specific server-side tools with `ServerSideTool(spec={...})`, including OpenRouter's web search implementation. +- Every tool call now has a unique `tool_call_id`, synthesized when the provider does not supply one. Tool implementations can {ref}`accept an llm_tool_call parameter ` to inspect the current call and its ID. +- Tools can {ref}`raise llm.PauseChain ` to pause execution for human approval or another external event. Chains can later {ref}`resume from a message history ending in unresolved tool calls `, without repeating calls that already have results. +- {ref}`Conversations that use configured tools ` can be continued with `llm -c` or `llm chat -c` without repeating the toolbox configuration. +- `llm tools` now shows constructor signatures and docstrings for {ref}`dynamic toolboxes `. Passing a toolbox specification instantiates it and lists its runtime-generated tools, while `llm tools --json` identifies dynamic toolboxes with a `"dynamic"` boolean. [#1580](https://github.com/simonw/llm/issues/1580) +- Models can now {ref}`declare the server-side tools they support ` using the instance-level `supported_server_side_tools` property and the new `llm.ServerSideTool` base class. Server-executed calls and results are captured as structured message parts, and `llm -c` restores configured server-side tools for continued conversations. [#1592](https://github.com/simonw/llm/issues/1592), [#1593](https://github.com/simonw/llm/pull/1593) +- {ref}`llm tools -m MODEL ` lists the server-side tools supported by that model. `llm models --json` returns model aliases, capability flags, attachment types and server-side tools, with option schemas included when combined with `--options`. +- {ref}`OpenAI-compatible Responses endpoints ` can use provider-specific server-side tools with `ServerSideTool(spec={...})`, including OpenRouter's web search implementation. See {ref}`0.32a3 ` for more detail on pausing, resuming and inspecting tool calls. ### New SQLite logging schema -LLM now logs prompts and responses using a new schema built around threads, turns and a **content-addressed message store**. Existing records in the legacy `responses` table are left untouched, and `llm logs` combines both generations of data. You can create a backup before upgrading using: +LLM now logs prompts and responses using a new schema built around threads, turns and a **content-addressed message store**. Existing records in the legacy `responses` table are left untouched, and `llm logs` combines both generations of data. You can {ref}`create a backup ` before upgrading using: llm logs backup logs-backup.db - Messages are stored once and referenced by their content hash, preserving structured text, reasoning, attachments and tool activity without duplicating repeated conversation history. See {ref}`the message store documentation `. - Raw provider payloads are stored in `turns.response_json`, {ref}`condensed ` using [condense-json](https://github.com/simonw/condense-json). `llm logs --json` expands them back to their original shape, and `LogStore.turn_response_json(turn_id)` returns them from Python. [#1586](https://github.com/simonw/llm/pull/1586) - Model plugins can define {ref}`json_replacements ` dictionaries to further improve payload compression. -- Full-text search, model and tool filters and conversation continuation work across both the legacy and new tables. Logs now record which configured toolbox instance supplied each tool. +- {ref}`Full-text search `, {ref}`model ` and {ref}`tool filters `, and {ref}`conversation views ` work across both the legacy and new tables. Logs now record which configured toolbox instance supplied each tool. - New {ref}`Response.log_to_db() ` Python API writes a response to a logs database. `llm prompt --json` outputs the same structured representation as `llm logs --json`, even when persistent logging is disabled. - New {ref}`message_tree SQL view ` renders conversation threads as indented text outlines for direct SQL exploration. - LLM now requires [sqlite-utils 4.0](https://sqlite-utils.datasette.io/en/stable/changelog.html#v4-0) or higher and no longer depends on `sqlite-migrate`. @@ -65,7 +63,7 @@ See {ref}`0.32rc1 ` for the detailed migration notes and complete lis - `llm logs status` now counts records in the new `threads` and `turns` tables, with legacy conversation and response counts shown separately when present. [#1590](https://github.com/simonw/llm/issues/1590) - `Response.to_dict()` now executes an unconsumed synchronous response before serializing it instead of producing an empty assistant message list. [#1590](https://github.com/simonw/llm/issues/1590) - `Response.from_dict()` now restores pending client-side tool calls so they can be inspected, executed or continued using `response.reply(tools=[...])`. [#1590](https://github.com/simonw/llm/issues/1590) -- `Response.reply()` now correctly passes attachments returned by tools to the next model call, for both synchronous and asynchronous responses. [#1590](https://github.com/simonw/llm/issues/1590) +- `Response.reply()` now correctly passes {ref}`attachments returned by tools ` to the next model call, for both synchronous and asynchronous responses. [#1590](https://github.com/simonw/llm/issues/1590) (v0_32_rc2)= ## 0.32rc2 (2026-07-30) diff --git a/docs/python-api.md b/docs/python-api.md index 33807d2a7..76e2cf37b 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -685,6 +685,8 @@ Event types are `"text"`, `"reasoning"`, `"tool_call_name"`, `"tool_call_args"`, Iterating against the response object itself (`for chunk in response`) yields only text strings — reasoning and tool-call events are filtered out. +(python-api-messages-reasoning)= + #### Hiding reasoning output Some model plugins can return visible reasoning text, exposed as `"reasoning"` events from `response.stream_events()` and assembled as `ReasoningPart` objects in `response.messages()`. From 847056dbea7c77fb17b7f6b045b98b50b756a693 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Aug 2026 10:02:42 -0700 Subject: [PATCH 244/258] Fix streamed server tool payload reconciliation Refresh server-side tool events from the final Responses payload so WebSearch sources and image results are not truncated. Add synchronous and asynchronous regression coverage. --- llm/default_plugins/openai_models.py | 47 ++++++++++- tests/test_openai_responses.py | 112 +++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 4 deletions(-) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 6854afbf9..93c50014d 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -2292,6 +2292,31 @@ def _server_tool_events(self, item, message_index): ) return events + def _refresh_server_tool_events(self, output, done_events): + """Replace streamed server-tool payloads with their final values. + + OpenAI can return incomplete sources, results or outputs on a + ``response.output_item.done`` event and then provide the complete + item on ``response.completed``. The response stores yielded + StreamEvent objects by reference, so updating their chunks here + corrects the assembled Parts without emitting duplicate events. + """ + for item in output or []: + item_id = getattr(item, "id", None) + prior_events = done_events.get(item_id) + if not prior_events: + continue + final_events = { + event.type: event + for event in self._server_tool_events( + item, prior_events[0].message_index + ) + } + for prior_event in prior_events: + final_event = final_events.get(prior_event.type) + if final_event is not None: + prior_event.chunk = final_event.chunk + def _non_streaming_output_events(self, output, response): """Translate a non-streaming Responses ``output`` item list into StreamEvents. Returns ``(events, had_reasoning)``. @@ -2460,6 +2485,7 @@ def execute( final_response_dict: dict[str, Any] | None = None reasoning_items_with_streamed_text = set() reasoning_done_events: dict[str, StreamEvent] = {} + server_tool_done_events: dict[str, list[StreamEvent]] = {} message_index = 0 seen_message = False for event in stream_obj: @@ -2556,8 +2582,15 @@ def execute( ) ) else: - yield from self._server_tool_events(item, message_index) + server_events = self._server_tool_events(item, message_index) + item_id = getattr(item, "id", None) + if item_id and server_events: + server_tool_done_events[item_id] = server_events + yield from server_events elif etype == "response.completed": + self._refresh_server_tool_events( + event.response.output, server_tool_done_events + ) final_response_dict = event.response.model_dump(warnings=False) if final_response_dict.get("usage"): usage = final_response_dict["usage"] @@ -2698,6 +2731,7 @@ async def execute( final_response_dict: dict[str, Any] | None = None reasoning_items_with_streamed_text = set() reasoning_done_events: dict[str, StreamEvent] = {} + server_tool_done_events: dict[str, list[StreamEvent]] = {} message_index = 0 seen_message = False async for event in stream_obj: @@ -2794,11 +2828,16 @@ async def execute( ) ) else: - for server_event in self._server_tool_events( - item, message_index - ): + server_events = self._server_tool_events(item, message_index) + item_id = getattr(item, "id", None) + if item_id and server_events: + server_tool_done_events[item_id] = server_events + for server_event in server_events: yield server_event elif etype == "response.completed": + self._refresh_server_tool_events( + event.response.output, server_tool_done_events + ) final_response_dict = event.response.model_dump(warnings=False) if final_response_dict.get("usage"): usage = final_response_dict["usage"] diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index ddba5ed40..e06e0f798 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -462,6 +462,60 @@ def _code_interpreter_stream(): ) +def _web_search_refresh_stream(): + def image_result(name): + return { + "type": "image_result", + "image_url": f"https://example.com/{name}.jpg", + "source_website_url": f"https://example.com/{name}", + "thumbnail_url": f"https://example.com/{name}-thumb.jpg", + "caption": f"{name.title()} image", + } + + def web_search_item(sources, results): + return { + "id": "ws_stream", + "type": "web_search_call", + "status": "completed", + "action": { + "type": "search", + "query": "example images", + "queries": ["example images"], + "sources": sources, + }, + "results": results, + } + + partial_sources = [{"type": "url", "url": "https://example.com/first"}] + final_sources = partial_sources + [ + {"type": "url", "url": "https://example.com/second"}, + ] + partial_results = [image_result("first")] + final_results = partial_results + [image_result("second")] + yield _responses_sse( + "response.output_item.done", + { + "output_index": 0, + "item": web_search_item(partial_sources, partial_results), + }, + ) + yield _responses_sse( + "response.output_text.delta", + { + "item_id": "msg_stream", + "output_index": 1, + "content_index": 0, + "delta": "done", + }, + ) + response_json = _text_response_json(text="done") + response_json["output"].insert(0, web_search_item(final_sources, final_results)) + yield _responses_sse( + "response.completed", + {"response": response_json}, + ) + + def _responses_reasoning_summary_stream(): yield _responses_sse( "response.reasoning_summary_text.delta", @@ -1399,6 +1453,64 @@ def test_code_interpreter_streaming_output_and_request(httpx_mock): assert "code_interpreter_call.outputs" in request_body["include"] +def _assert_web_search_streaming_uses_final_payload(response, messages): + parts = [part for message in messages for part in message.parts] + server_parts = [ + part + for part in parts + if isinstance(part, (llm.parts.ToolCallPart, llm.parts.ToolResultPart)) + ] + assert [type(part).__name__ for part in server_parts] == [ + "ToolCallPart", + "ToolResultPart", + ] + tool_call, tool_result = server_parts + final_item = next( + item + for item in response.response_json["output"] + if item["type"] == "web_search_call" + ) + assert len(final_item["action"]["sources"]) == 2 + assert len(final_item["results"]) == 2 + assert tool_call.arguments == final_item["action"] + assert json.loads(tool_result.output) == final_item["results"] + + +def test_web_search_streaming_refreshes_from_final_payload(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + stream=IteratorStream(_web_search_refresh_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + response = llm.get_model("gpt-5.6-luna").prompt( + "Search", + tools=[WebSearch(include_sources=True, include_results=True)], + key="test", + ) + + assert response.text() == "done" + _assert_web_search_streaming_uses_final_payload(response, response.messages()) + + +@pytest.mark.asyncio +async def test_async_web_search_streaming_refreshes_from_final_payload(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + stream=IteratorStream(_web_search_refresh_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + response = llm.get_async_model("gpt-5.6-luna").prompt( + "Search", + tools=[WebSearch(include_sources=True, include_results=True)], + key="test", + ) + + assert await response.text() == "done" + _assert_web_search_streaming_uses_final_payload(response, await response.messages()) + + def test_server_tool_parts_not_replayed_as_function_calls(): from llm.parts import Message, TextPart, ToolCallPart, ToolResultPart From 09e279d3550746474cd64a4f240157abaa82dee6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Aug 2026 10:06:15 -0700 Subject: [PATCH 245/258] Note about visible reasoning traces, refs #1590 --- docs/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 36f0d4e0e..554f7335b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,7 +3,7 @@ (v0_32)= ## 0.32 (2026-08-03) -LLM 0.32 is a major, backwards-compatible update to the way prompts, responses, tools and logs are represented. It adds structured messages and parts throughout the Python API, adopts the OpenAI Responses API for reasoning-capable models, substantially expands control over pausable and resumable tool loops and introduces a new content-addressed SQLite logging schema. +LLM 0.32 is a major, backwards-compatible update to the way prompts, responses, tools and logs are represented. It adds structured messages and parts throughout the Python API, adopts the OpenAI Responses API for reasoning-capable models, substantially expands control over pausable and resumable tool loops and introduces a new content-addressed SQLite logging schema. Reasoning traces are now displayed on standard error by the `llm` command, for models that support them. ### Structured messages and richer responses From 76abd6095682702804ceba2c6a51d43dd76b1ef5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Aug 2026 10:11:50 -0700 Subject: [PATCH 246/258] Release 0.32 Refs #346, #468, #506, #716, #770, #813, #867, #894, #937, #938, #1019, #1024, #1033, #1046, #1067, #1080, #1092, #1278, #1314, #1426, #1432, #1433, #1435, #1441, #1442, #1466, #1467, #1469, #1478, #1480, #1481, #1482, #1483, #1486, #1487, #1489, #1511, #1515, #1521, #1544, #1553, #1554, #1555, #1556, #1557, #1558, #1560, #1562, #1563, #1565, #1566, #1568, #1571, #1574, #1575, #1576, #1577, #1579, #1580, #1585, #1586, #1588, #1590, #1591, #1593 Closes #1590 --- docs/changelog.md | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 554f7335b..b110bc488 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,7 +1,7 @@ # Changelog (v0_32)= -## 0.32 (2026-08-03) +## 0.32 (2026-08-04) LLM 0.32 is a major, backwards-compatible update to the way prompts, responses, tools and logs are represented. It adds structured messages and parts throughout the Python API, adopts the OpenAI Responses API for reasoning-capable models, substantially expands control over pausable and resumable tool loops and introduces a new content-addressed SQLite logging schema. Reasoning traces are now displayed on standard error by the `llm` command, for models that support them. diff --git a/pyproject.toml b/pyproject.toml index 049a9e61a..8ec3191a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm" -version = "0.32rc2" +version = "0.32" description = "CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine." readme = { file = "README.md", content-type = "text/markdown" } authors = [ From 71a4815a2cd1957febcac5f566fb13a483fc30f5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Aug 2026 10:15:10 -0700 Subject: [PATCH 247/258] Removed repeat links to #1590 --- docs/changelog.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index b110bc488..6ac73c22b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -60,10 +60,10 @@ See {ref}`0.32rc1 ` for the detailed migration notes and complete lis - Fixed streamed OpenAI Responses API calls recording two different ciphertexts of the same reasoning: the part's `encrypted_content` was harvested from the `response.output_item.done` event while `response_json` came from `response.completed`, and OpenAI encrypts per event. Reasoning metadata is now re-emitted from the final payload, so both records agree on one blob. - Attachments loaded from URLs now follow up to three redirects when detecting their content type or fetching their bytes. Thanks, [Ojas Sharma](https://github.com/ojassharma7). [#1046](https://github.com/simonw/llm/issues/1046), [#1579](https://github.com/simonw/llm/pull/1579) - Fixed a bug where `llm openai endpoint --schema` was ignored if the selected template also defined a schema. Thanks, [ikatyal2110](https://github.com/ikatyal2110). [#1588](https://github.com/simonw/llm/pull/1588) -- `llm logs status` now counts records in the new `threads` and `turns` tables, with legacy conversation and response counts shown separately when present. [#1590](https://github.com/simonw/llm/issues/1590) -- `Response.to_dict()` now executes an unconsumed synchronous response before serializing it instead of producing an empty assistant message list. [#1590](https://github.com/simonw/llm/issues/1590) -- `Response.from_dict()` now restores pending client-side tool calls so they can be inspected, executed or continued using `response.reply(tools=[...])`. [#1590](https://github.com/simonw/llm/issues/1590) -- `Response.reply()` now correctly passes {ref}`attachments returned by tools ` to the next model call, for both synchronous and asynchronous responses. [#1590](https://github.com/simonw/llm/issues/1590) +- `llm logs status` now counts records in the new `threads` and `turns` tables, with legacy conversation and response counts shown separately when present. +- `Response.to_dict()` now executes an unconsumed synchronous response before serializing it instead of producing an empty assistant message list. +- `Response.from_dict()` now restores pending client-side tool calls so they can be inspected, executed or continued using `response.reply(tools=[...])`. +- `Response.reply()` now correctly passes {ref}`attachments returned by tools ` to the next model call, for both synchronous and asynchronous responses. (v0_32_rc2)= ## 0.32rc2 (2026-07-30) From 9733d343f6f65b44462e417093f3e3718476f702 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Aug 2026 10:31:18 -0700 Subject: [PATCH 248/258] Ran Cog --- docs/fragments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/fragments.md b/docs/fragments.md index 2f1e0df90..366505b55 100644 --- a/docs/fragments.md +++ b/docs/fragments.md @@ -30,7 +30,7 @@ from importlib.metadata import version llm_version = version("llm") cog.out(f'The URL will be fetched with the user-agent `llm/{llm_version} (https://llm.datasette.io/)`.') ]]]--> -The URL will be fetched with the user-agent `llm/0.32rc2 (https://llm.datasette.io/)`. +The URL will be fetched with the user-agent `llm/0.32 (https://llm.datasette.io/)`. The `-f` option can be used multiple times to combine together multiple fragments. From 9d963a8810683e77bb0c989614035d1fcee53d1d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Aug 2026 13:23:29 -0700 Subject: [PATCH 249/258] Remove pointless 'No server-side tools for' message from llm tools -m --- llm/cli.py | 23 +++++++++-------------- tests/test_cli_openai_models.py | 1 - 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index 22c9b58c1..376bc8522 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -2893,20 +2893,15 @@ def introspect_tools(toolbox): click.echo( textwrap.indent(tool_info["description"].strip(), " ") + "\n" ) - if model is not None: - if server_side_tools: - click.echo( - f"Server-side tools for {model.model_id} " - "(executed by the provider):\n" - ) - for tool_info in server_side_tools: - click.echo(f"{tool_info['name']}{tool_info['signature']}\n") - if tool_info["description"]: - click.echo( - textwrap.indent(tool_info["description"], " ") + "\n" - ) - else: - click.echo(f"No server-side tools for {model.model_id}.") + if model is not None and server_side_tools: + click.echo( + f"Server-side tools for {model.model_id} " + "(executed by the provider):\n" + ) + for tool_info in server_side_tools: + click.echo(f"{tool_info['name']}{tool_info['signature']}\n") + if tool_info["description"]: + click.echo(textwrap.indent(tool_info["description"], " ") + "\n") @cli.group( diff --git a/tests/test_cli_openai_models.py b/tests/test_cli_openai_models.py index 27ebe5fd8..b763e1bf7 100644 --- a/tests/test_cli_openai_models.py +++ b/tests/test_cli_openai_models.py @@ -487,7 +487,6 @@ def test_tools_list_for_model_with_no_server_side_tools(): result = runner.invoke(cli, ["tools", "-m", "chatgpt"]) assert result.exit_code == 0 - assert "No server-side tools for gpt-3.5-turbo.\n" in result.output json_result = runner.invoke(cli, ["tools", "-m", "chatgpt", "--json"]) assert json_result.exit_code == 0 From 163c08e05f864753edcf3de7f8a9984ce7fb61b6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Aug 2026 15:08:32 -0700 Subject: [PATCH 250/258] Fix for flaky CI warning test --- tests/test_openai_endpoint.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py index d6844d9b0..3356fcc08 100644 --- a/tests/test_openai_endpoint.py +++ b/tests/test_openai_endpoint.py @@ -889,7 +889,7 @@ def test_endpoint_responses_api_tools(httpx_mock, user_path): ] -def test_endpoint_responses_api_raw_server_side_tool(httpx_mock, user_path, recwarn): +def test_endpoint_responses_api_raw_server_side_tool(httpx_mock, user_path): base_url = "https://raw-tools.example.test/v1" response_payload = _responses_payload("Search complete") response_payload["output"].insert( @@ -938,7 +938,6 @@ def test_endpoint_responses_api_raw_server_side_tool(httpx_mock, user_path, recw assert not (user_path / "logs.db").exists() request_body = json.loads(httpx_mock.get_requests()[0].content) assert request_body["tools"] == [tool_spec] - assert not recwarn.list def test_endpoint_reads_one_off_prompt_from_stdin(httpx_mock, user_path): From 8493825f877a7e6b460882a610ef60c4ea5caa1a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Aug 2026 20:36:36 -0700 Subject: [PATCH 251/258] Replace references to outdated Gemini models --- docs/changelog.md | 2 +- docs/plugins/advanced-model-plugins.md | 2 +- docs/usage.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 6ac73c22b..2e6887dcf 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -625,7 +625,7 @@ response = model.prompt( ``` Plugins that provide alternative models can support attachments, see {ref}`advanced-model-plugins-attachments` for details. -The latest **[llm-claude-3](https://github.com/simonw/llm-claude-3)** plugin now supports attachments for Anthropic's Claude 3 and 3.5 models. The **[llm-gemini](https://github.com/simonw/llm-gemini)** plugin supports attachments for Google's Gemini 1.5 models. +The latest **[llm-claude-3](https://github.com/simonw/llm-claude-3)** plugin now supports attachments for Anthropic's Claude 3 and 3.5 models. The **[llm-gemini](https://github.com/simonw/llm-gemini)** plugin supports attachments for Google's `gemini-flash-latest` model. Also in this release: OpenAI models now record their `"usage"` data in the database even when the response was streamed. These records can be viewed using `llm logs --json`. [#591](https://github.com/simonw/llm/issues/591) diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index 9e6564ceb..9c4b5a83e 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -215,7 +215,7 @@ Server-side tool calls returned by the provider should use the `server_executed= ## Attachments for multi-modal models -Models such as GPT-4o, Claude 3.5 Sonnet and Google's Gemini 1.5 are multi-modal: they accept input in the form of images and maybe even audio, video and other formats. +Models such as GPT-4o, Claude 3.5 Sonnet and Google's `gemini-flash-latest` are multi-modal: they accept input in the form of images and maybe even audio, video and other formats. LLM calls these **attachments**. Models can specify the types of attachments they accept and then implement special code in the `.execute()` method to handle them. diff --git a/docs/usage.md b/docs/usage.md index 93352b316..b33313ea8 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -65,7 +65,7 @@ You can also {ref}`configure default options ` (usage-attachments)= ### Attachments -Some models are multi-modal, which means they can accept input in more than just text. GPT-4o and GPT-4o mini can accept images, and models such as Google Gemini 1.5 can accept audio and video as well. +Some models are multi-modal, which means they can accept input in more than just text. GPT-4o and GPT-4o mini can accept images, and models such as Google's `gemini-flash-latest` can accept audio and video as well. LLM calls these **attachments**. You can pass attachments using the `-a` option like this: From cca86ef02a564e49fe14fc10d80ca41c75cb11bf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Aug 2026 20:38:33 -0700 Subject: [PATCH 252/258] Drop LLama, add other modern models --- README.md | 2 +- docs/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 247086a69..1dcee9860 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ cog.out(readme_markdown) [![Discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord-llm) [![Homebrew](https://img.shields.io/homebrew/installs/dy/llm?color=yellow&label=homebrew&logo=homebrew)](https://formulae.brew.sh/formula/llm) -A CLI tool and Python library for interacting with **OpenAI**, **Anthropic’s Claude**, **Google’s Gemini**, **Meta’s Llama** and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine. +A CLI tool and Python library for interacting with **OpenAI**, **Anthropic’s Claude**, **Google’s Gemini**, **Qwen**, **Gemma**, **Kimi**, **DeepSeek**, **Mistral**, and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine. Watch **[Language models on the command-line](https://www.youtube.com/watch?v=QUXQNi6jQ30)** on YouTube for a demo or [read the accompanying detailed notes](https://simonwillison.net/2024/Jun/17/cli-language-models/). diff --git a/docs/index.md b/docs/index.md index b144b9319..939ef3807 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,7 +8,7 @@ [![Discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord-llm) [![Homebrew](https://img.shields.io/homebrew/installs/dy/llm?color=yellow&label=homebrew&logo=homebrew)](https://formulae.brew.sh/formula/llm) -A CLI tool and Python library for interacting with **OpenAI**, **Anthropic's Claude**, **Google's Gemini**, **Meta's Llama** and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine. +A CLI tool and Python library for interacting with **OpenAI**, **Anthropic’s Claude**, **Google’s Gemini**, **Qwen**, **Gemma**, **Kimi**, **DeepSeek**, **Mistral**, and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine. Watch **[Language models on the command-line](https://www.youtube.com/watch?v=QUXQNi6jQ30)** on YouTube for a demo or [read the accompanying detailed notes](https://simonwillison.net/2024/Jun/17/cli-language-models/). From f3f4f29c6e5e623c3776cbf6455dda2345638214 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 5 Aug 2026 07:12:56 -0700 Subject: [PATCH 253/258] Capture UnknownModelError, not ValueError Closes #1523 --- llm/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm/cli.py b/llm/cli.py index 376bc8522..96cd01e98 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -3492,7 +3492,7 @@ def embed_multi( collection_obj = Collection( collection, db=db, model_id=model or get_default_embedding_model() ) - except ValueError: + except UnknownModelError: raise click.ClickException( "You need to specify an embedding model (no default model is set)" ) From 4ed661ebf98c375a228fcc0dcf05e27c2a4617b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Aug 2026 17:41:31 -0700 Subject: [PATCH 254/258] Fix embed-multi without a default model When no embedding model is configured, open an existing collection without trying to create it so its stored model can be reused. Translate a missing collection into the intended CLI error without restoring the broad ValueError handling. --- llm/cli.py | 5 +++-- tests/test_embed_cli.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/llm/cli.py b/llm/cli.py index 96cd01e98..a9968160d 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -3488,11 +3488,12 @@ def embed_multi( for alias, attach_path in attach: db.attach(alias, attach_path) + model_id = model or get_default_embedding_model() try: collection_obj = Collection( - collection, db=db, model_id=model or get_default_embedding_model() + collection, db=db, model_id=model_id, create=model_id is not None ) - except UnknownModelError: + except (Collection.DoesNotExist, UnknownModelError): raise click.ClickException( "You need to specify an embedding model (no default model is set)" ) diff --git a/tests/test_embed_cli.py b/tests/test_embed_cli.py index 11b67f56c..693955610 100644 --- a/tests/test_embed_cli.py +++ b/tests/test_embed_cli.py @@ -688,6 +688,21 @@ def test_default_embed_model_errors(user_path, default_is_set, command): assert db["embeddings"].count == 1 +def test_embed_multi_existing_collection_without_default(user_path): + db = sqlite_utils.Database(str(user_path / "embeddings.db")) + Collection("example", db, model_id="embed-demo") + + result = CliRunner().invoke( + cli, + ["embed-multi", "example", "-"], + input="id,name\n1,hello", + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert db["embeddings"].count == 1 + + def test_duplicate_content_embedded_only_once(embed_demo): # content_hash should avoid embedding the same content twice # per collection From 5a86b35e486462b15dddd3f6276b9449a7f5435d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Aug 2026 17:20:31 -0700 Subject: [PATCH 255/258] Add reasoning_summary option for Responses endpoints OpenAI-compatible Responses endpoints can require an explicit reasoning.summary setting before they return reasoning summaries. Expose auto, concise, and detailed as a Responses-only model option while preserving the existing opt-in default for arbitrary endpoints. Closes #1600 --- docs/changelog.md | 5 ++ docs/other-models.md | 14 +++++- docs/usage.md | 27 ++++++++++ llm/default_plugins/openai_models.py | 36 ++++++++++++-- tests/test_openai_endpoint.py | 74 ++++++++++++++++++++++++++++ tests/test_openai_responses.py | 66 ++++++++++++++++++++++++- 6 files changed, 217 insertions(+), 5 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 2e6887dcf..f9c9e16cc 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,10 @@ # Changelog +(unreleased)= +## Unreleased + +- Reasoning-capable Responses API models now support a `reasoning_summary` option with `auto`, `concise`, and `detailed` values. This can be used with {ref}`llm openai endpoint --responses `. [#1600](https://github.com/simonw/llm/issues/1600) + (v0_32)= ## 0.32 (2026-08-04) diff --git a/docs/other-models.md b/docs/other-models.md index a2a8dd695..662d7af4e 100644 --- a/docs/other-models.md +++ b/docs/other-models.md @@ -106,7 +106,19 @@ llm openai endpoint https://example.com/v1 \ "Solve this problem" ``` -The command does not send reasoning-specific request fields by default and does not request a reasoning summary. Those fields are only added when `reasoning_effort` is used. An endpoint that does not support the option will return its own API error. +Responses API endpoints can also be asked for a visible reasoning summary using `-o reasoning_summary`, with a value of `auto`, `concise`, or `detailed`: + +```bash +llm openai endpoint https://example.com/v1 \ + --responses \ + -m model-id \ + -o reasoning_summary auto \ + "Solve this problem" +``` + +This maps to the Responses API `reasoning.summary` request field. `auto` asks the endpoint for the most detailed summary available for that model. Use `-R` or `--hide-reasoning` to suppress summary generation for a request. + +The command does not send reasoning-specific request fields by default and does not request a reasoning summary unless one of these options is used. An endpoint that does not support an option will return its own API error. Use `-T` or `--tool` to make an installed LLM tool available to the model, or `--functions` to load Python functions from an inline code block or file: diff --git a/docs/usage.md b/docs/usage.md index b33313ea8..f4271cddf 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -888,6 +888,9 @@ OpenAI Responses: o1 supported values are low, medium, and high. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning in a response. + reasoning_summary: str + Requests a summary of the model's reasoning. Supported values are + auto, concise, and detailed. service_tier: str The processing tier to use for this request - for example 'fast' for Fast mode (faster responses at a higher price) or 'flex' for slower, @@ -915,6 +918,7 @@ OpenAI Responses: o1-2024-12-17 chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp @@ -939,6 +943,7 @@ OpenAI Responses: o3-mini chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str service_tier: str Features: - streaming @@ -962,6 +967,7 @@ OpenAI Responses: o3 chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp @@ -987,6 +993,7 @@ OpenAI Responses: o4-mini chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str service_tier: str Attachment types: application/pdf, image/gif, image/jpeg, image/png, image/webp @@ -1012,6 +1019,7 @@ OpenAI Responses: gpt-5 chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1038,6 +1046,7 @@ OpenAI Responses: gpt-5-mini chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1064,6 +1073,7 @@ OpenAI Responses: gpt-5-nano chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1090,6 +1100,7 @@ OpenAI Responses: gpt-5-2025-08-07 chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1116,6 +1127,7 @@ OpenAI Responses: gpt-5-mini-2025-08-07 chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1142,6 +1154,7 @@ OpenAI Responses: gpt-5-nano-2025-08-07 chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1168,6 +1181,7 @@ OpenAI Responses: gpt-5.1 chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1194,6 +1208,7 @@ OpenAI Responses: gpt-5.2 chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1220,6 +1235,7 @@ OpenAI Responses: gpt-5.2-chat-latest chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1246,6 +1262,7 @@ OpenAI Responses: gpt-5.4 chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1272,6 +1289,7 @@ OpenAI Responses: gpt-5.4-2026-03-05 chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1298,6 +1316,7 @@ OpenAI Responses: gpt-5.4-mini chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1324,6 +1343,7 @@ OpenAI Responses: gpt-5.4-mini-2026-03-17 chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1350,6 +1370,7 @@ OpenAI Responses: gpt-5.4-nano chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1376,6 +1397,7 @@ OpenAI Responses: gpt-5.4-nano-2026-03-17 chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1402,6 +1424,7 @@ OpenAI Responses: gpt-5.5 chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1428,6 +1451,7 @@ OpenAI Responses: gpt-5.5-2026-04-23 chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1454,6 +1478,7 @@ OpenAI Responses: gpt-5.6-sol chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1480,6 +1505,7 @@ OpenAI Responses: gpt-5.6-terra chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: @@ -1506,6 +1532,7 @@ OpenAI Responses: gpt-5.6-luna chat_completions: boolean image_detail: str reasoning_effort: str + reasoning_summary: str verbosity: str service_tier: str Attachment types: diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 93c50014d..94cec5938 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -966,6 +966,12 @@ class ReasoningEffortEnum(str, Enum): max = "max" +class ReasoningSummaryEnum(str, Enum): + auto = "auto" + concise = "concise" + detailed = "detailed" + + class VerbosityEnum(str, Enum): low = "low" medium = "medium" @@ -995,6 +1001,7 @@ def enum_values_sentence(enum_class): def build_options_class( *, reasoning=False, + reasoning_summary=False, verbosity=False, image_detail_original=False, chat_completions=False, @@ -1049,6 +1056,18 @@ def build_options_class( default=None, ), ) + if reasoning_summary: + reasoning_summary_values = enum_values_sentence(ReasoningSummaryEnum) + fields["reasoning_summary"] = ( + ReasoningSummaryEnum | None, + Field( + description=( + "Requests a summary of the model's reasoning. Supported values " + f"are {reasoning_summary_values}." + ), + default=None, + ), + ) if verbosity: fields["verbosity"] = ( VerbosityEnum | None, @@ -1325,6 +1344,9 @@ def build_kwargs(self, prompt, stream): json_object = kwargs.pop("json_object", None) kwargs.pop("image_detail", None) kwargs.pop("chat_completions", None) + # Responses models reuse their Options object when explicitly routed + # through the Chat Completions compatibility path. + kwargs.pop("reasoning_summary", None) if "max_tokens" not in kwargs and self.default_max_tokens is not None: kwargs["max_tokens"] = self.default_max_tokens if json_object: @@ -2020,6 +2042,7 @@ def _build_responses_kwargs(self, prompt, stream): opts.pop("image_detail", None) max_tokens = opts.pop("max_tokens", None) reasoning_effort = opts.pop("reasoning_effort", None) + reasoning_summary = opts.pop("reasoning_summary", None) verbosity = opts.pop("verbosity", None) temperature = opts.pop("temperature", None) top_p = opts.pop("top_p", None) @@ -2038,8 +2061,11 @@ def _build_responses_kwargs(self, prompt, stream): kwargs["seed"] = seed if self._reasoning: reasoning = {} - if self._reasoning_summary and not getattr(prompt, "hide_reasoning", False): - reasoning["summary"] = "auto" + if not getattr(prompt, "hide_reasoning", False): + if reasoning_summary is not None: + reasoning["summary"] = reasoning_summary + elif self._reasoning_summary: + reasoning["summary"] = "auto" if reasoning_effort: reasoning["effort"] = reasoning_effort if reasoning: @@ -2092,7 +2118,9 @@ def _finalize_responses_kwargs(self, prompt, stream, instructions=None): kwargs["instructions"] = instructions kwargs["store"] = False if self._reasoning and ( - self._reasoning_summary or getattr(prompt.options, "reasoning_effort", None) + self._reasoning_summary + or getattr(prompt.options, "reasoning_summary", None) + or getattr(prompt.options, "reasoning_effort", None) ): include = kwargs.setdefault("include", []) if "reasoning.encrypted_content" not in include: @@ -2440,6 +2468,7 @@ def __init__( # always available on Responses-routed models. self.Options = build_options_class( reasoning=reasoning, + reasoning_summary=reasoning, verbosity=verbosity, image_detail_original=image_detail_original, chat_completions=True, @@ -2683,6 +2712,7 @@ def __init__( self._service_tier = service_tier self.Options = build_options_class( reasoning=reasoning, + reasoning_summary=reasoning, verbosity=verbosity, image_detail_original=image_detail_original, chat_completions=True, diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py index 3356fcc08..d263edf10 100644 --- a/tests/test_openai_endpoint.py +++ b/tests/test_openai_endpoint.py @@ -1,6 +1,7 @@ import base64 import json +import pytest import sqlite_utils from click.testing import CliRunner from pytest_httpx import IteratorStream @@ -727,6 +728,79 @@ def test_endpoint_responses_api(httpx_mock, user_path): } +@pytest.mark.parametrize("reasoning_summary", ("auto", "concise", "detailed")) +def test_endpoint_responses_reasoning_summary_option( + httpx_mock, user_path, reasoning_summary +): + base_url = "https://responses-summary.example.test/v1" + httpx_mock.add_response( + method="POST", + url=f"{base_url}/responses", + json=_responses_payload("Hello from Responses"), + headers={"Content-Type": "application/json"}, + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Hello", + "-m", + "test-model", + "--responses", + "--no-stream", + "-o", + "reasoning_summary", + reasoning_summary, + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "Hello from Responses\n" + assert not (user_path / "logs.db").exists() + assert json.loads(httpx_mock.get_requests()[0].content) == { + "input": [{"role": "user", "content": "Hello"}], + "include": ["reasoning.encrypted_content"], + "model": "test-model", + "reasoning": {"summary": reasoning_summary}, + "store": False, + "stream": False, + } + + +def test_endpoint_responses_reasoning_summary_option_rejects_invalid_value( + httpx_mock, user_path +): + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + "https://responses-summary.example.test/v1", + "Hello", + "-m", + "test-model", + "--responses", + "--no-stream", + "-o", + "reasoning_summary", + "verbose", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 1 + assert result.output == ( + "Error: reasoning_summary\n" + " Input should be 'auto', 'concise' or 'detailed'\n" + ) + assert not httpx_mock.get_requests() + assert not (user_path / "logs.db").exists() + + def test_endpoint_responses_api_attachment(httpx_mock, user_path): base_url = "https://responses-attachments.example.test/v1" httpx_mock.add_response( diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index e06e0f798..bff2909cd 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -565,10 +565,18 @@ def _responses_reasoning_summary_stream(): def test_responses_model_is_registered(): + from llm.default_plugins.openai_models import Chat + model = llm.get_model("gpt-5.5") assert "Responses" in type(model).__name__ # The chat_completions opt-out option must be exposed. assert "chat_completions" in model.Options.model_fields + assert "reasoning_summary" in model.Options.model_fields + assert "reasoning_summary" in llm.get_async_model("gpt-5.5").Options.model_fields + assert ( + "reasoning_summary" + not in Chat("reasoning-chat-model", reasoning=True).Options.model_fields + ) def test_chat_completions_opt_out_dispatches_to_chat(httpx_mock): @@ -593,8 +601,16 @@ def test_chat_completions_opt_out_dispatches_to_chat(httpx_mock): headers={"Content-Type": "application/json"}, ) model = llm.get_model("gpt-5.5") - response = model.prompt("hello", stream=False, chat_completions=True, key="test") + response = model.prompt( + "hello", + stream=False, + chat_completions=True, + reasoning_summary="detailed", + key="test", + ) assert response.text() == "hi from chat" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert "reasoning_summary" not in request_body def test_default_routes_to_responses_endpoint(httpx_mock): @@ -891,6 +907,37 @@ class FakePrompt: assert kwargs["reasoning"] == {"summary": "auto"} +@pytest.mark.parametrize("reasoning_summary", ("auto", "concise", "detailed")) +def test_responses_kwargs_explicit_reasoning_summary(reasoning_summary): + model = llm.get_model("gpt-5.5") + options = model.Options(reasoning_summary=reasoning_summary) + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + kwargs = model._build_responses_kwargs(p, stream=False) + assert kwargs["reasoning"] == {"summary": reasoning_summary} + + +def test_async_responses_kwargs_explicit_reasoning_summary(): + model = llm.get_async_model("gpt-5.5") + options = model.Options(reasoning_summary="concise") + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + kwargs = model._build_responses_kwargs(p, stream=False) + assert kwargs["reasoning"] == {"summary": "concise"} + + def test_responses_kwargs_omits_reasoning_summary_when_hide_reasoning(): model = llm.get_model("gpt-5.5") options = model.Options(reasoning_effort="low") @@ -907,6 +954,23 @@ class FakePrompt: assert kwargs["reasoning"] == {"effort": "low"} +def test_responses_kwargs_omits_explicit_reasoning_summary_when_hide_reasoning(): + model = llm.get_model("gpt-5.5") + options = model.Options(reasoning_summary="detailed") + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + p.hide_reasoning = True + kwargs = model._finalize_responses_kwargs(p, stream=False) + assert "reasoning" not in kwargs + assert kwargs["include"] == ["reasoning.encrypted_content"] + + def test_responses_kwargs_omits_empty_reasoning_when_hide_reasoning(): model = llm.get_model("gpt-5.5") options = model.Options() From c4eaae3d4b038e5ac249c0cb90a0e1eb1420dbdc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Aug 2026 18:12:21 -0700 Subject: [PATCH 256/258] 'just docs PORT' to run on a port other than 8000 --- Justfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Justfile b/Justfile index 626ff09c4..f85fddc58 100644 --- a/Justfile +++ b/Justfile @@ -27,10 +27,10 @@ @cog: uv run cog -r -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" docs/**/*.md docs/*.md README.md -# Serve live docs on localhost:8000 -@docs: cog +# Serve live docs on localhost (default port: 8000) +@docs port="8000": cog rm -rf docs/_build - cd docs && uv run make livehtml + cd docs && uv run make livehtml SPHINXOPTS="--port {{port}}" # Apply Black @black: From c410af0774d93d00c727bab22f33619c316d6148 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Aug 2026 18:12:49 -0700 Subject: [PATCH 257/258] Preserve metadata-only reasoning parts Refs https://github.com/simonw/llm-anthropic/issues/81 --- llm/models.py | 2 +- llm/parts.py | 7 +++ tests/test_logs_store.py | 25 +++++++++ tests/test_parts.py | 115 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 148 insertions(+), 1 deletion(-) diff --git a/llm/models.py b/llm/models.py index 6e616b904..dc66c5c39 100644 --- a/llm/models.py +++ b/llm/models.py @@ -1400,7 +1400,7 @@ def _build_message_parts(self) -> list[list[Any]]: elif fam_first == "reasoning": text = "".join(e.chunk for e in evs) redacted = any(e.redacted for e in evs) - if text or redacted: + if text or redacted or pm_merged: built.append( ( mi, diff --git a/llm/parts.py b/llm/parts.py index 58f4dcea2..4debba51f 100644 --- a/llm/parts.py +++ b/llm/parts.py @@ -332,6 +332,13 @@ class StreamEvent: `signature`, Gemini `thoughtSignature`, OpenAI `encrypted_content`) that must be echoed back on the next request; the framework merges it onto the finalized Part (last non-None wins per top-level key). + A reasoning event with an empty `chunk` and `redacted=False` still + assembles into a ReasoningPart when it carries provider_metadata — + the metadata-only form used for opaque reasoning state such as an + Anthropic omitted-thinking signature or `redacted_thinking` data. + Unlike `redacted=True` markers, metadata-only reasoning Parts are + not hoisted: they keep their emitted position, because providers + require opaque blocks replayed in their original order. `message_index` is for providers that emit multiple assistant messages in a single response (OpenAI Responses server-side tool diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py index 719d7ca7f..cd19399ba 100644 --- a/tests/test_logs_store.py +++ b/tests/test_logs_store.py @@ -209,6 +209,31 @@ def test_reasoning_including_redacted(self, store): ] assert round_trip(store, messages) == messages + def test_metadata_only_opaque_reasoning_round_trips(self, store): + # Anthropic omitted thinking (empty text + signature) and + # redacted_thinking (opaque data) as adjacent, distinct parts: + # exact metadata and ordering must survive the database so the + # blocks can be replayed byte-for-byte on continuation. + messages = [ + llm.user("Think"), + llm.assistant( + ReasoningPart( + text="", + provider_metadata={ + "anthropic": {"type": "thinking", "signature": "sig-a"} + }, + ), + ReasoningPart( + text="", + provider_metadata={ + "anthropic": {"type": "redacted_thinking", "data": "blob-b"} + }, + ), + ToolCallPart(name="search", arguments={"q": "x"}, tool_call_id="tc1"), + ), + ] + assert round_trip(store, messages) == messages + def test_interleaved_parts_keep_their_order(self, store): # The ordering the old schema could not express: reasoning, a # tool call, more reasoning, then text, all in one message. diff --git a/tests/test_parts.py b/tests/test_parts.py index ffe3b6c08..ffbf51295 100644 --- a/tests/test_parts.py +++ b/tests/test_parts.py @@ -510,6 +510,121 @@ def test_redacted_reasoning_event_default_redacted_is_false(self): ev = llm.parts.StreamEvent(type="reasoning", chunk="thinking") assert ev.redacted is False + def test_metadata_only_reasoning_event_emits_part(self, mock_model): + # Anthropic display:"omitted" thinking arrives as an empty + # thinking block whose signature carries the encrypted + # reasoning state. The signature-only event must still become + # a ReasoningPart or the signature is lost from history. + events = [ + llm.parts.StreamEvent( + type="reasoning", + chunk="", + provider_metadata={"anthropic": {"signature": "sig-omitted"}}, + ), + llm.parts.StreamEvent(type="text", chunk="hi"), + ] + mock_model.enqueue(events) + response = mock_model.prompt("x") + response.text() + parts = response.messages()[0].parts + assert parts == [ + llm.parts.ReasoningPart( + text="", + redacted=False, + provider_metadata={"anthropic": {"signature": "sig-omitted"}}, + ), + llm.parts.TextPart(text="hi"), + ] + assert "sig-omitted" not in response.text() + + def test_metadata_only_reasoning_part_is_not_hoisted(self, mock_model): + # Unlike redacted=True markers, metadata-only reasoning keeps + # its position: Anthropic requires thinking/redacted_thinking + # blocks replayed in their original order. + events = [ + llm.parts.StreamEvent(type="text", chunk="hello", part_index=0), + llm.parts.StreamEvent( + type="reasoning", + chunk="", + part_index=1, + provider_metadata={ + "anthropic": {"type": "redacted_thinking", "data": "opaque"} + }, + ), + llm.parts.StreamEvent(type="text", chunk="bye", part_index=2), + ] + mock_model.enqueue(events) + response = mock_model.prompt("x") + response.text() + parts = response.messages()[0].parts + assert parts == [ + llm.parts.TextPart(text="hello"), + llm.parts.ReasoningPart( + text="", + redacted=False, + provider_metadata={ + "anthropic": {"type": "redacted_thinking", "data": "opaque"} + }, + ), + llm.parts.TextPart(text="bye"), + ] + + def test_adjacent_reasoning_blocks_with_explicit_index_stay_distinct( + self, mock_model + ): + # Explicit part_index keeps adjacent provider blocks separate: + # a visible signed thinking block followed by a redacted one + # must not merge (each keeps its own metadata and boundaries). + events = [ + llm.parts.StreamEvent(type="reasoning", chunk="visible ", part_index=0), + llm.parts.StreamEvent( + type="reasoning", + chunk="", + part_index=0, + provider_metadata={"anthropic": {"signature": "sig-1"}}, + ), + llm.parts.StreamEvent( + type="reasoning", + chunk="", + part_index=1, + provider_metadata={ + "anthropic": {"type": "redacted_thinking", "data": "blob"} + }, + ), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=2), + ] + mock_model.enqueue(events) + response = mock_model.prompt("x") + response.text() + parts = response.messages()[0].parts + assert parts == [ + llm.parts.ReasoningPart( + text="visible ", + redacted=False, + provider_metadata={"anthropic": {"signature": "sig-1"}}, + ), + llm.parts.ReasoningPart( + text="", + redacted=False, + provider_metadata={ + "anthropic": {"type": "redacted_thinking", "data": "blob"} + }, + ), + llm.parts.TextPart(text="answer"), + ] + + def test_empty_reasoning_group_without_metadata_is_dropped(self, mock_model): + # No text, no redacted marker, no metadata: nothing worth + # preserving, so no Part is built. + events = [ + llm.parts.StreamEvent(type="reasoning", chunk=""), + llm.parts.StreamEvent(type="text", chunk="hi"), + ] + mock_model.enqueue(events) + response = mock_model.prompt("x") + response.text() + assert response.messages()[0].parts == [llm.parts.TextPart(text="hi")] + class TestPartIndexAutoAllocation: """When part_index is None (the default), the framework groups From 5b332ec027300603086d5df26b462fc7c8dd83f5 Mon Sep 17 00:00:00 2001 From: "Agusti F." <6601142+agustif@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:19:54 +0200 Subject: [PATCH 258/258] LM Studio in plugin directory !stable-docs --- docs/plugins/directory.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/plugins/directory.md b/docs/plugins/directory.md index f26193999..2dd8d806d 100644 --- a/docs/plugins/directory.md +++ b/docs/plugins/directory.md @@ -15,6 +15,7 @@ These plugins all help you run LLMs directly on your own computer: - **[llm-mlc](https://github.com/simonw/llm-mlc)** can run local models released by the [MLC project](https://mlc.ai/mlc-llm/), including models that can take advantage of the GPU on Apple Silicon M1/M2 devices. - **[llm-gpt4all](https://github.com/simonw/llm-gpt4all)** adds support for various models released by the [GPT4All](https://gpt4all.io/) project that are optimized to run locally on your own machine. These models include versions of Vicuna, Orca, Falcon and MPT - here's [a full list of models](https://observablehq.com/@simonw/gpt4all-models). - **[llm-mpt30b](https://github.com/simonw/llm-mpt30b)** adds support for the [MPT-30B](https://huggingface.co/mosaicml/mpt-30b) local model. +- **[llm-lmstudio](https://github.com/agustif/llm-lmstudio)** provides access to local models using [LM Studio](https://lmstudio.ai/), (plugin-directory-remote-apis)= ## Remote APIs