diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py index 8c9c255103d53..f547ef3711600 100644 --- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py +++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py @@ -880,6 +880,135 @@ async def test_astream(self, model: BaseChatModel) -> None: assert len(full.content_blocks) == 1 assert full.content_blocks[0]["type"] == "text" + def test_invoke_with_model_override(self, model: BaseChatModel) -> None: + """Test that model name can be overridden at invoke time via kwargs. + + This enables dynamic model selection without creating new instances, + which is useful for fallback strategies, A/B testing, or cost optimization. + + Test is skipped if `supports_model_override` is `False`. + + ??? question "Troubleshooting" + + If this test fails, ensure that your `_generate` method passes + `**kwargs` through to the API request payload in a way that allows + the `model` parameter to be overridden. + + For example: + ```python + def _get_request_payload(self, ..., **kwargs) -> dict: + return { + "model": self.model, + ... + **kwargs, # kwargs should come last to allow overrides + } + ``` + """ + if not self.supports_model_override: + pytest.skip("Model override not supported.") + + override_model = self.model_override_value + if not override_model: + pytest.skip("model_override_value not specified.") + + result = model.invoke("Hello", model=override_model) + assert result is not None + assert isinstance(result, AIMessage) + + # Verify the overridden model was used + model_name = result.response_metadata.get("model_name") + assert model_name is not None, "model_name not found in response_metadata" + assert override_model in model_name, ( + f"Expected model '{override_model}' but got '{model_name}'" + ) + + async def test_ainvoke_with_model_override(self, model: BaseChatModel) -> None: + """Test that model name can be overridden at ainvoke time via kwargs. + + Test is skipped if `supports_model_override` is `False`. + + ??? question "Troubleshooting" + + See troubleshooting for `test_invoke_with_model_override`. + """ + if not self.supports_model_override: + pytest.skip("Model override not supported.") + + override_model = self.model_override_value + if not override_model: + pytest.skip("model_override_value not specified.") + + result = await model.ainvoke("Hello", model=override_model) + assert result is not None + assert isinstance(result, AIMessage) + + # Verify the overridden model was used + model_name = result.response_metadata.get("model_name") + assert model_name is not None, "model_name not found in response_metadata" + assert override_model in model_name, ( + f"Expected model '{override_model}' but got '{model_name}'" + ) + + def test_stream_with_model_override(self, model: BaseChatModel) -> None: + """Test that model name can be overridden at stream time via kwargs. + + Test is skipped if `supports_model_override` is `False`. + + ??? question "Troubleshooting" + + See troubleshooting for `test_invoke_with_model_override`. + """ + if not self.supports_model_override: + pytest.skip("Model override not supported.") + + override_model = self.model_override_value + if not override_model: + pytest.skip("model_override_value not specified.") + + full: AIMessageChunk | None = None + for chunk in model.stream("Hello", model=override_model): + assert isinstance(chunk, AIMessageChunk) + full = chunk if full is None else full + chunk + + assert full is not None + + # Verify the overridden model was used + model_name = full.response_metadata.get("model_name") + assert model_name is not None, "model_name not found in response_metadata" + assert override_model in model_name, ( + f"Expected model '{override_model}' but got '{model_name}'" + ) + + async def test_astream_with_model_override(self, model: BaseChatModel) -> None: + """Test that model name can be overridden at astream time via kwargs. + + Test is skipped if `supports_model_override` is `False`. + + ??? question "Troubleshooting" + + See troubleshooting for `test_invoke_with_model_override`. + """ + if not self.supports_model_override: + pytest.skip("Model override not supported.") + + override_model = self.model_override_value + if not override_model: + pytest.skip("model_override_value not specified.") + + full: AIMessageChunk | None = None + async for chunk in model.astream("Hello", model=override_model): + assert isinstance(chunk, AIMessageChunk) + full = chunk if full is None else full + chunk + + assert full is not None + + # Verify the overridden model was used + model_name = full.response_metadata.get("model_name") + assert model_name is not None, "model_name not found in response_metadata" + assert override_model in model_name, ( + f"Expected model '{override_model}' but got '{model_name}'" + ) + def test_batch(self, model: BaseChatModel) -> None: """Test to verify that `model.batch([messages])` works. diff --git a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py index e339e5b1592c4..5460623bd9aaa 100644 --- a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py +++ b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py @@ -250,6 +250,28 @@ def supported_usage_metadata_details( """ return {"invoke": [], "stream": []} + @property + def supports_model_override(self) -> bool: + """Whether the model supports overriding the model name at runtime. + + Defaults to `True`. + + If `True`, the model accepts a `model` kwarg in `invoke()`, `stream()`, + etc. that overrides the model specified at initialization. + + This enables dynamic model selection without creating new instances. + """ + return True + + @property + def model_override_value(self) -> str | None: + """Alternative model name to use when testing model override. + + Should return a valid model name that differs from the default model. + Required if `supports_model_override` is `True`. + """ + return None + class ChatModelUnitTests(ChatModelTests): '''Base class for chat model unit tests. @@ -672,6 +694,36 @@ def supports_anthropic_inputs(self) -> bool: Only needs to be overridden if these details are supplied. + ??? info "`supports_model_override`" + + Boolean property indicating whether the chat model supports overriding the + model name at runtime via kwargs. + + If `True`, the model accepts a `model` kwarg in `invoke()`, `stream()`, etc. + that overrides the model specified at initialization. This enables dynamic + model selection without creating new chat model instances. + + Defaults to `False`. + + ```python + @property + def supports_model_override(self) -> bool: + return True + ``` + + ??? info "`model_override_value`" + + Alternative model name to use when testing model override. + + Should return a valid model name that differs from the default model. + Required if `supports_model_override` is `True`. + + ```python + @property + def model_override_value(self) -> str: + return "gpt-4o-mini" # e.g. if default is "gpt-4o" + ``` + ??? info "`enable_vcr_tests`" Property controlling whether to enable select tests that rely on