From 7d93ab5db1b99036f255b4968b1284c8af773770 Mon Sep 17 00:00:00 2001 From: Yamac Ay Date: Thu, 10 Sep 2026 15:51:14 +0200 Subject: [PATCH 01/11] fix(sap): introduce cache_control redone --- .../gen_ai_hub/orchestration_v2/__init__.py | 5 +- .../orchestration_v2/models/__init__.py | 8 ++- .../orchestration_v2/models/cache_control.py | 52 +++++++++++++++++++ .../models/multimodal_items.py | 10 +++- .../orchestration_v2/models/response.py | 24 ++++++++- .../orchestration_v2/models/tools.py | 5 ++ .../orchestration_v2/test_flat_import.py | 9 ++-- 7 files changed, 104 insertions(+), 9 deletions(-) create mode 100644 packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py diff --git a/packages/gen/gen_ai_hub/orchestration_v2/__init__.py b/packages/gen/gen_ai_hub/orchestration_v2/__init__.py index 260f0c7e..d378b8f7 100644 --- a/packages/gen/gen_ai_hub/orchestration_v2/__init__.py +++ b/packages/gen/gen_ai_hub/orchestration_v2/__init__.py @@ -3,6 +3,9 @@ from .exceptions import OrchestrationError, OrchestrationErrorList __all__ = [ + # cache_control + "CacheControl", + # azure_content_filter "AzureContentFilter", "AzureContentSafetyInput", "AzureContentSafetyOutput", "AzureThreshold", @@ -46,7 +49,7 @@ "ImageDetailLevel", "TextPart", "ImageUrl", "ImagePart", "ContentPart", "ImageItem", # response - "PromptTokensDetails", "CompletionTokensDetails", "TokenUsage", "GenericModuleResult", "TopLogprob", + "CacheCreationTokenDetails", "PromptTokensDetails", "CompletionTokensDetails", "TokenUsage", "GenericModuleResult", "TopLogprob", "ChatCompletionTokenLogprob", "ChoiceLogprobs", "LLMChoice", "StreamFunctionObject", "StreamToolCall", "StreamDelta", "StreamLLMChoice", "Citation", "LLMModuleResult", "StreamLLMModuleResult", "ModuleResults", "StreamModuleResults", "SAPAPIError", "SAPAPIErrorStreaming", "CompletionPostResponse", diff --git a/packages/gen/gen_ai_hub/orchestration_v2/models/__init__.py b/packages/gen/gen_ai_hub/orchestration_v2/models/__init__.py index 790b0b75..44e231e7 100644 --- a/packages/gen/gen_ai_hub/orchestration_v2/models/__init__.py +++ b/packages/gen/gen_ai_hub/orchestration_v2/models/__init__.py @@ -1,4 +1,5 @@ from .azure_content_filter import AzureContentSafetyInput, AzureContentSafetyOutput, AzureContentFilter, AzureThreshold +from .cache_control import CacheControl from .config import (ModuleConfig, OrchestrationConfig, OrchestrationConfigReference, CompletionRequestConfigurationReferenceByIdConfigRef, CompletionRequestConfigurationReferenceByNameScenarioVersionConfigRef) @@ -20,7 +21,7 @@ from .message import (SystemMessage, UserMessage, AssistantMessage, ToolChatMessage, DeveloperChatMessage, ChatMessage, ResponseChatMessage, FunctionCall, MessageToolCall) from .multimodal_items import ImageDetailLevel, TextPart, ImageUrl, ImagePart, ContentPart, ImageItem -from .response import (PromptTokensDetails, CompletionTokensDetails, TokenUsage, GenericModuleResult, TopLogprob, +from .response import (CacheCreationTokenDetails, PromptTokensDetails, CompletionTokensDetails, TokenUsage, GenericModuleResult, TopLogprob, ChatCompletionTokenLogprob, ChoiceLogprobs, LLMChoice, StreamFunctionObject, StreamToolCall, StreamDelta, StreamLLMChoice, Citation, LLMModuleResult, StreamLLMModuleResult, ModuleResults, StreamModuleResults, SAPAPIError, SAPAPIErrorStreaming, CompletionPostResponse, @@ -36,6 +37,9 @@ __all__ = [ + # cache_control + "CacheControl", + # azure_content_filter "AzureContentFilter", "AzureContentSafetyInput", "AzureContentSafetyOutput", "AzureThreshold", @@ -79,7 +83,7 @@ "ImageDetailLevel", "TextPart", "ImageUrl", "ImagePart", "ContentPart", "ImageItem", # response - "PromptTokensDetails", "CompletionTokensDetails", "TokenUsage", "GenericModuleResult", "TopLogprob", + "CacheCreationTokenDetails", "PromptTokensDetails", "CompletionTokensDetails", "TokenUsage", "GenericModuleResult", "TopLogprob", "ChatCompletionTokenLogprob", "ChoiceLogprobs", "LLMChoice", "StreamFunctionObject", "StreamToolCall", "StreamDelta", "StreamLLMChoice", "Citation", "LLMModuleResult", "StreamLLMModuleResult", "ModuleResults", "StreamModuleResults", "SAPAPIError", "SAPAPIErrorStreaming", "CompletionPostResponse", diff --git a/packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py b/packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py new file mode 100644 index 00000000..348d7d95 --- /dev/null +++ b/packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py @@ -0,0 +1,52 @@ +"""Cache control for prompt caching on supported Anthropic and Amazon Nova models.""" +from typing import Any, Dict, Literal, Optional + +from pydantic import model_serializer + +from gen_ai_hub.orchestration_v2.models.base import ABCBaseModel as BaseModel + + +class CacheControl(BaseModel): + """Marks a message content block or tool definition for prompt caching. + + When attached to a content block, the model stores intermediate computation + results for that content and reuses them on subsequent requests within the + TTL window, reducing both latency and token costs. + + Supported models: + - Anthropic Claude: system and user content blocks; tools. + - Amazon Nova: system and user content blocks only (no tools, no TTL). + + Attach ``CacheControl`` directly to a content block (``TextPart``, ``ImagePart``) or + to a ``ChatCompletionTool``. For the "last-block shorthand" pass ``cache_control`` to + ``OrchestrationService.run()``; it calls ``apply_cache_control_to_last_message()`` + automatically. + + Args: + type: Always ``"ephemeral"``. Only value supported by the API. + ttl: Cache duration. ``"5m"`` (default) or ``"1h"`` (select Anthropic + models only). Omit for Amazon Nova or when the default is sufficient. + + Example:: + + from gen_ai_hub.orchestration_v2.models.cache_control import CacheControl + from gen_ai_hub.orchestration_v2.models.multimodal_items import TextPart + + block = TextPart(text="Long context...", cache_control=CacheControl()) + block_1h = TextPart(text="Long context...", cache_control=CacheControl(ttl="1h")) + """ + + type: Literal["ephemeral"] = "ephemeral" + ttl: Optional[Literal["5m", "1h"]] = None + + @model_serializer(mode="wrap") + def serialize_wire_format(self, handler: Any) -> Dict[str, Any]: + """Serialize to the wire format, omitting ``ttl`` when not set. + + :return: ``{"type": "ephemeral"}`` or ``{"type": "ephemeral", "ttl": ""}`` + :rtype: dict + """ + data: Dict[str, Any] = handler(self) + if data.get("ttl") is None: + data.pop("ttl", None) + return data diff --git a/packages/gen/gen_ai_hub/orchestration_v2/models/multimodal_items.py b/packages/gen/gen_ai_hub/orchestration_v2/models/multimodal_items.py index 87855b24..9f17c893 100644 --- a/packages/gen/gen_ai_hub/orchestration_v2/models/multimodal_items.py +++ b/packages/gen/gen_ai_hub/orchestration_v2/models/multimodal_items.py @@ -12,6 +12,7 @@ from pydantic.main import IncEx from gen_ai_hub.orchestration_v2.models.base import ABCBaseModel as BaseModel +from gen_ai_hub.orchestration_v2.models.cache_control import CacheControl class ImageDetailLevel(Enum): @@ -38,9 +39,13 @@ class TextPart(BaseModel): text: The string content of the text part. type: The type identifier, defaulting to "text". + + cache_control: Optional cache control settings for prompt caching. + Only supported for Anthropic Claude and Amazon Nova models. """ text: str type_: Literal["text"] = Field(default="text", alias="type") + cache_control: Optional[CacheControl] = Field(default=None, exclude=False) class ImageUrl(BaseModel): @@ -56,7 +61,6 @@ class ImageUrl(BaseModel): detail: Optional[ImageDetailLevel] = None -# @dataclass class ImagePart(BaseModel): """ Represents an image segment within a multimodal content block. @@ -65,9 +69,13 @@ class ImagePart(BaseModel): image_url: An `ImageUrl` object containing the image's location and detail level. type: The type identifier, defaulting to "image_url". + + cache_control: Optional cache control settings for prompt caching. + Only supported for Anthropic Claude models. """ image_url: ImageUrl type_: Literal["image_url"] = Field(default="image_url", alias="type") + cache_control: Optional[CacheControl] = Field(default=None, exclude=False) ContentPart = Union[TextPart, ImagePart] diff --git a/packages/gen/gen_ai_hub/orchestration_v2/models/response.py b/packages/gen/gen_ai_hub/orchestration_v2/models/response.py index ca398edf..cc6c6449 100644 --- a/packages/gen/gen_ai_hub/orchestration_v2/models/response.py +++ b/packages/gen/gen_ai_hub/orchestration_v2/models/response.py @@ -27,16 +27,35 @@ class ResponseBaseModel(BaseModel): ) +class CacheCreationTokenDetails(ResponseBaseModel): + """ + Per-TTL breakdown of tokens written to the prompt cache. + + Present only when cache_control includes an explicit ttl value. + + Attributes: + ephemeral_5m_input_tokens: Tokens cached with a 5-minute TTL. + ephemeral_1h_input_tokens: Tokens cached with a 1-hour TTL. + """ + ephemeral_5m_input_tokens: Optional[int] = None + ephemeral_1h_input_tokens: Optional[int] = None + + class PromptTokensDetails(ResponseBaseModel): """ Represents the details of prompt tokens used in a specific operation. Attributes: audio_tokens (Optional[int]): Audio input tokens present in the prompt. - cached_tokens (Optional[int]): Cached tokens present in the prompt. + cached_tokens (Optional[int]): Tokens read from the prompt cache (cache hit). + cache_creation_tokens (Optional[int]): Tokens written to the prompt cache (cache miss). + cache_creation_token_details (Optional[CacheCreationTokenDetails]): Per-TTL + breakdown of cache writes. Present only when an explicit ttl was used. """ audio_tokens: Optional[int] = None cached_tokens: Optional[int] = None + cache_creation_tokens: Optional[int] = None + cache_creation_token_details: Optional[CacheCreationTokenDetails] = None class CompletionTokensDetails(ResponseBaseModel): """ @@ -385,7 +404,8 @@ class OrchestrationResponseWithRetries(CompletionPostResponse): """ retries: int = 0 -__all__ = ["PromptTokensDetails", +__all__ = ["CacheCreationTokenDetails", + "PromptTokensDetails", "CompletionTokensDetails", "TokenUsage", "GenericModuleResult", diff --git a/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py b/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py index ea4811bf..e17bc105 100644 --- a/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py +++ b/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py @@ -11,6 +11,7 @@ from pydantic import Field from gen_ai_hub.orchestration_v2.models.base import ABCBaseModel as BaseModel +from gen_ai_hub.orchestration_v2.models.cache_control import CacheControl def python_type_to_json_type(py_type): @@ -71,10 +72,14 @@ class ChatCompletionTool(BaseModel): Args: type (Literal["function"]): The type of the tool. Currently, only function is supported. + + cache_control: Optional cache control settings for prompt caching. + Supported for Anthropic Claude models only. Not supported for Amazon Nova. """ type_: Literal["function"] = Field(default="function", alias="type", description="The type of the tool. Currently, only function is supported.") + cache_control: Optional[CacheControl] = None class FunctionObject(BaseModel): diff --git a/packages/gen/tests/orchestration_v2/test_flat_import.py b/packages/gen/tests/orchestration_v2/test_flat_import.py index 7606ffac..ca877b41 100644 --- a/packages/gen/tests/orchestration_v2/test_flat_import.py +++ b/packages/gen/tests/orchestration_v2/test_flat_import.py @@ -1,4 +1,7 @@ expected = { + # cache_control + "CacheControl", + # azure_content_filter "AzureContentFilter", "AzureContentSafetyInput", "AzureContentSafetyOutput", "AzureThreshold", @@ -46,7 +49,7 @@ "ChatCompletionTokenLogprob", "ChoiceLogprobs", "LLMChoice", "StreamFunctionObject", "StreamToolCall", "StreamDelta", "StreamLLMChoice", "Citation", "LLMModuleResult", "StreamLLMModuleResult", "ModuleResults", "StreamModuleResults", "SAPAPIError", "SAPAPIErrorStreaming", "CompletionPostResponse", - "StreamCompletionPostResponse", "ErrorResponse", "ErrorResponseStreaming", "OrchestrationResponseWithRetries", + "StreamCompletionPostResponse", "ErrorResponse", "ErrorResponseStreaming", "OrchestrationResponseWithRetries", "CacheCreationTokenDetails", # response_format "ResponseFormatText", "ResponseFormatJsonObject", "ResponseFormatJsonSchema", "JSONResponseSchema", @@ -72,8 +75,8 @@ "OrchestrationService", # Exceptions - "OrchestrationError", "OrchestrationErrorList" - } + "OrchestrationError", "OrchestrationErrorList", + } def test_flat_import_all(): From 1854bfcefb17b68ef849947ef8f50c31d80e0b24 Mon Sep 17 00:00:00 2001 From: Yamac Ay Date: Thu, 10 Sep 2026 15:52:46 +0200 Subject: [PATCH 02/11] e2e tests --- .../orchestration_v2/test_cache_control.py | 282 ++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 packages/gen/integration_tests/orchestration_v2/test_cache_control.py diff --git a/packages/gen/integration_tests/orchestration_v2/test_cache_control.py b/packages/gen/integration_tests/orchestration_v2/test_cache_control.py new file mode 100644 index 00000000..786d1845 --- /dev/null +++ b/packages/gen/integration_tests/orchestration_v2/test_cache_control.py @@ -0,0 +1,282 @@ +""" +Unit and integration tests for prompt caching (cache_control) via Orchestration V2. + +Caching is supported for Anthropic Claude and Amazon Nova models. +The test targets anthropic--claude-4.6-sonnet (1024-token minimum, 5m and 1h TTLs). + +Wire path: + ai-sdk-python -> SAP AI Core /v2/completion -> SAP LiteLLM fork -> Anthropic API + +cache_control is serialized as a plain JSON key on the content block or tool dict by +Pydantic model_dump(). The SAP LiteLLM fork translates it into Anthropic's native +prompt-caching format. + +The spec defines cache_control on three schema-level attachment points: + - TextContent.cache_control (TextPart in py) + - UserChatMessageContentItem.cache_control (TextPart / ImagePart in py) + - ChatCompletionTool.cache_control + +Response fields (from SAP AI Core orchestration docs): + prompt_tokens_details.cached_tokens -- tokens read from cache (hit) + prompt_tokens_details.cache_creation_tokens -- tokens written to cache (miss) + prompt_tokens_details.cache_creation_token_details.ephemeral_5m_input_tokens + prompt_tokens_details.cache_creation_token_details.ephemeral_1h_input_tokens +""" +import unittest + +from gen_ai_hub.orchestration_v2.models.cache_control import CacheControl +from gen_ai_hub.orchestration_v2.models.config import OrchestrationConfig, ModuleConfig +from gen_ai_hub.orchestration_v2.models.llm_model_details import LLMModelDetails +from gen_ai_hub.orchestration_v2.models.message import SystemMessage, UserMessage +from gen_ai_hub.orchestration_v2.models.multimodal_items import TextPart, ImagePart, ImageUrl +from gen_ai_hub.orchestration_v2.models.template import Template, PromptTemplatingModuleConfig +from gen_ai_hub.orchestration_v2.models.tools import ChatCompletionTool, FunctionTool, FunctionObject +from gen_ai_hub.orchestration_v2.service import OrchestrationService +from integration_tests.orchestration_v2.test_base import OrchestrationServiceTestBase +from integration_tests.test_helpers import retry_on_429_or_503 + +# Must exceed the 1024-token minimum for claude-4.6-sonnet cache points. +_LONG_SYSTEM_PROMPT = ( + "You are a helpful assistant with deep knowledge of European history. " + "Below is a detailed reference text that you must use to answer questions accurately.\n\n" + + ( + "The Roman Empire was one of the largest empires in ancient history. " + "At its height under Emperor Trajan in 117 AD, it covered over 5 million " + "square kilometres and held 70 million people, roughly 21 percent of the " + "world's population at the time. The empire's longevity — nearly five " + "centuries in the west and fifteen in the east — shaped the languages, " + "laws, religions, and borders of modern Europe. Latin evolved into the " + "Romance languages: Italian, Spanish, Portuguese, French, and Romanian. " + "Roman law underlies most continental legal systems today. Christianity, " + "adopted as the state religion under Theodosius I in 380 AD, spread " + "throughout the empire and became the dominant faith of Europe. " + "The fall of the Western Roman Empire in 476 AD, when the Germanic " + "chieftain Odoacer deposed the last emperor Romulus Augustulus, marks " + "the conventional boundary between ancient and medieval history. " + "The Eastern Roman Empire, known as the Byzantine Empire, continued " + "for nearly a thousand more years until the fall of Constantinople to " + "the Ottoman Turks in 1453. Byzantine culture preserved classical Greek " + "and Roman learning through the Dark Ages and transmitted it to the " + "Renaissance. The Silk Road trade routes connecting Rome to China " + "facilitated the exchange of goods, diseases, and ideas across Eurasia. " + "Roman engineering achievements — aqueducts, roads, concrete construction, " + "and underfloor heating — were not equalled in Europe for over a millennium " + "after the empire's fall. The Colosseum, completed in 80 AD, could seat " + "50,000 to 80,000 spectators and hosted gladiatorial contests, animal " + "hunts, and public executions for four centuries. " + ) * 4 # repeat to comfortably exceed 1024 tokens +) + +_LLM = LLMModelDetails( + name="anthropic--claude-4.6-sonnet", + params={"max_tokens": 64, "temperature": 0.0}, +) + + +def _config(messages, tools=None): + return OrchestrationConfig( + modules=ModuleConfig( + prompt_templating=PromptTemplatingModuleConfig( + prompt=Template(template=messages, tools=tools), + model=_LLM, + ) + ) + ) + + +class TestCacheControlSerialization(unittest.TestCase): + """Unit tests: verify cache_control serializes correctly without a network call.""" + + # ------------------------------------------------------------------ + # CacheControl model + # ------------------------------------------------------------------ + + def test_cache_control_default_ttl_omits_key(self): + """CacheControl() with no TTL serializes to {"type": "ephemeral"}.""" + d = CacheControl().model_dump(by_alias=True) + self.assertEqual(d, {"type": "ephemeral"}) + self.assertNotIn("ttl", d) + + def test_cache_control_5m_ttl(self): + """CacheControl(ttl="5m") serializes with ttl field.""" + d = CacheControl(ttl="5m").model_dump(by_alias=True) + self.assertEqual(d, {"type": "ephemeral", "ttl": "5m"}) + + def test_cache_control_1h_ttl(self): + """CacheControl(ttl="1h") serializes with ttl field.""" + d = CacheControl(ttl="1h").model_dump(by_alias=True) + self.assertEqual(d, {"type": "ephemeral", "ttl": "1h"}) + + # ------------------------------------------------------------------ + # TextPart with cache_control + # ------------------------------------------------------------------ + + def test_text_part_with_cache_control(self): + """TextPart with cache_control serializes the cache_control block.""" + part = TextPart(text="hello", cache_control=CacheControl()) + d = part.model_dump(by_alias=True, exclude_none=True) + self.assertEqual(d["type"], "text") + self.assertEqual(d["text"], "hello") + self.assertEqual(d["cache_control"], {"type": "ephemeral"}) + + def test_text_part_without_cache_control_omits_key(self): + """TextPart without cache_control does not emit the key.""" + part = TextPart(text="hello") + d = part.model_dump(by_alias=True, exclude_none=True) + self.assertNotIn("cache_control", d) + + def test_text_part_with_1h_ttl(self): + """TextPart with CacheControl(ttl='1h') serializes the ttl field.""" + part = TextPart(text="hello", cache_control=CacheControl(ttl="1h")) + d = part.model_dump(by_alias=True, exclude_none=True) + self.assertEqual(d["cache_control"], {"type": "ephemeral", "ttl": "1h"}) + + # ------------------------------------------------------------------ + # ImagePart with cache_control + # ------------------------------------------------------------------ + + def test_image_part_with_cache_control(self): + """ImagePart with cache_control serializes the cache_control block.""" + part = ImagePart( + image_url=ImageUrl(url="https://example.com/img.png"), + cache_control=CacheControl(), + ) + d = part.model_dump(by_alias=True, exclude_none=True) + self.assertEqual(d["type"], "image_url") + self.assertEqual(d["cache_control"], {"type": "ephemeral"}) + + def test_image_part_without_cache_control_omits_key(self): + """ImagePart without cache_control does not emit the key.""" + part = ImagePart(image_url=ImageUrl(url="https://example.com/img.png")) + d = part.model_dump(by_alias=True, exclude_none=True) + self.assertNotIn("cache_control", d) + + def test_image_part_with_1h_ttl(self): + """ImagePart with CacheControl(ttl='1h') serializes the ttl field.""" + part = ImagePart( + image_url=ImageUrl(url="https://example.com/img.png"), + cache_control=CacheControl(ttl="1h"), + ) + d = part.model_dump(by_alias=True, exclude_none=True) + self.assertEqual(d["cache_control"], {"type": "ephemeral", "ttl": "1h"}) + + # ------------------------------------------------------------------ + # ChatCompletionTool with cache_control + # ------------------------------------------------------------------ + + def test_tool_cache_control_serialized(self): + """cache_control on a ChatCompletionTool appears at the tool level.""" + tool = FunctionTool( + function=FunctionObject( + name="classify", + description="Classify input.", + parameters={"type": "object", "properties": {}}, + ), + cache_control=CacheControl(), + ) + d = tool.model_dump(by_alias=True, exclude_none=True) + self.assertEqual(d["cache_control"], {"type": "ephemeral"}) + + def test_tool_without_cache_control_omits_key(self): + """A tool without cache_control does not emit the key.""" + tool = FunctionTool( + function=FunctionObject( + name="classify", + description="Classify input.", + parameters={"type": "object", "properties": {}}, + ), + ) + d = tool.model_dump(by_alias=True, exclude_none=True) + self.assertNotIn("cache_control", d) + + def test_tool_with_1h_ttl(self): + """cache_control with ttl='1h' on a tool serializes the ttl field.""" + tool = FunctionTool( + function=FunctionObject( + name="classify", + description="Classify input.", + parameters={"type": "object", "properties": {}}, + ), + cache_control=CacheControl(ttl="1h"), + ) + d = tool.model_dump(by_alias=True, exclude_none=True) + self.assertEqual(d["cache_control"], {"type": "ephemeral", "ttl": "1h"}) + + +class TestPromptCachingLive(OrchestrationServiceTestBase): + """Live integration tests against the SAP AI Core orchestration V2 service.""" + + def setUp(self): + super().setUp() + self.service = OrchestrationService(self.api_url) + + # ------------------------------------------------------------------ + # 1. Cache MISS on first call — cache breakpoint on TextPart directly + # ------------------------------------------------------------------ + + @retry_on_429_or_503(max_retries=3, initial_delay=2.0, backoff_factor=2.0) + def test_cache_miss_on_first_call(self): + """First call with cache_control on the last TextPart block returns + non-zero cache token activity. + + cache_creation_tokens > 0 on a true miss; cached_tokens > 0 when the + cache entry is already warm from a previous run. Either proves the + cache_control breakpoint was accepted by the server. + """ + config = _config([ + SystemMessage(content=[TextPart(text=_LONG_SYSTEM_PROMPT, cache_control=CacheControl())]), + UserMessage(content="In one word: what language did Romans speak?"), + ]) + response = self.service.run(config=config) + details = response.final_result.usage.prompt_tokens_details + self.assertIsNotNone(details) + cache_active = (details.cache_creation_tokens or 0) + (details.cached_tokens or 0) + self.assertGreater( + cache_active, 0, + f"Expected cache activity (cache_creation_tokens or cached_tokens > 0), " + f"got: {details}", + ) + + # ------------------------------------------------------------------ + # 2. Cache HIT on repeated call + # ------------------------------------------------------------------ + + @retry_on_429_or_503(max_retries=3, initial_delay=2.0, backoff_factor=2.0) + def test_cache_hit_on_repeated_call(self): + """Second call with the same cache breakpoint produces cached_tokens > 0.""" + config = _config([ + SystemMessage(content=[TextPart(text=_LONG_SYSTEM_PROMPT, cache_control=CacheControl())]), + UserMessage(content="In one word: what language did Romans speak?"), + ]) + self.service.run(config=config) # populate cache + response = self.service.run(config=config) + details = response.final_result.usage.prompt_tokens_details + self.assertIsNotNone(details) + self.assertGreater( + details.cached_tokens, 0, + "Expected cached_tokens > 0 on second call (cache hit).", + ) + + # ------------------------------------------------------------------ + # 3. Explicit 1h TTL via TextPart + # ------------------------------------------------------------------ + + @retry_on_429_or_503(max_retries=3, initial_delay=2.0, backoff_factor=2.0) + def test_explicit_ttl_1h_via_text_part(self): + """Attaching CacheControl(ttl='1h') directly to a TextPart returns + cache_creation_token_details with ephemeral_1h_input_tokens.""" + config = _config([ + SystemMessage(content=[TextPart(text=_LONG_SYSTEM_PROMPT, cache_control=CacheControl(ttl="1h"))]), + UserMessage(content="Name the last Western Roman emperor."), + ]) + response = self.service.run(config=config) + details = response.final_result.usage.prompt_tokens_details + self.assertIsNotNone(details) + self.assertIsNotNone( + details.cache_creation_token_details, + "Expected cache_creation_token_details when ttl='1h' is used.", + ) + + +if __name__ == "__main__": + unittest.main() From e16cbf6d1960fd76582d7a3c2887af1f151e2745 Mon Sep 17 00:00:00 2001 From: Yamac Ay Date: Thu, 10 Sep 2026 18:14:42 +0200 Subject: [PATCH 03/11] tests fixed and rmd duplicate class attribute --- .../orchestration_v2/models/tools.py | 41 ++---- .../orchestration_v2/test_cache_control.py | 135 +----------------- .../orchestration_v2/test_cache_control_v2.py | 117 +++++++++++++++ 3 files changed, 132 insertions(+), 161 deletions(-) create mode 100644 packages/gen/tests/orchestration_v2/test_cache_control_v2.py diff --git a/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py b/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py index e17bc105..b8ef1979 100644 --- a/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py +++ b/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py @@ -67,37 +67,24 @@ def python_type_to_json_type(py_type): class ChatCompletionTool(BaseModel): - """ - Base class for all chat completion tools. + """A tool the model may call, identified by type ``"function"``. Args: - type (Literal["function"]): The type of the tool. Currently, only function is supported. - - cache_control: Optional cache control settings for prompt caching. - Supported for Anthropic Claude models only. Not supported for Amazon Nova. + cache_control: Prompt-caching directive. Supported on Anthropic Claude only; + not supported on Amazon Nova. """ - type_: Literal["function"] = Field(default="function", - alias="type", - description="The type of the tool. Currently, only function is supported.") + type_: Literal["function"] = Field(default="function", alias="type") cache_control: Optional[CacheControl] = None class FunctionObject(BaseModel): - """ - Represents a function. - Args: - name (str): The name of the function to be called. Must be a-z, A-Z, 0-9, - or contain underscores and dashes, with a maximum length of 64. - - description (str): A description of what the function does, used by the model - to choose when and how to call the function. + """A function definition used inside a ``FunctionTool``. - parameters (dict): The parameters the functions accepts, described as a JSON Schema object. - Omitting parameters defines a function with an empty parameter list. - - strict (bool, optional): Whether to enable strict schema adherence when generating the function call. - If set to true, the model will follow the exact schema defined in the parameters field. - Only a subset of JSON Schema is supported when strict is true. Defaults to False. + Args: + name: Function name. Must match ``^[a-zA-Z0-9_-]+$``, max 64 chars. + description: What the function does; used by the model to decide when to call it. + parameters: JSON Schema object describing accepted parameters. + strict: When ``True``, the model follows the schema exactly. Defaults to ``False``. """ description: Optional[str] = None name: str @@ -107,15 +94,11 @@ class FunctionObject(BaseModel): class FunctionTool(ChatCompletionTool): - """ - Represents a function tool for OpenAI-like function calling. + """A callable function tool for OpenAI-style function calling. Args: - type (Literal["function"]): The type of the tool. Currently, only function is supported. - - function (FunctionObject): The function to be called. + function: The function definition (name, description, parameters). """ - type_: Literal["function"] = Field(default="function", alias="type") function: FunctionObject def execute(self, **kwargs: Any) -> Any: diff --git a/packages/gen/integration_tests/orchestration_v2/test_cache_control.py b/packages/gen/integration_tests/orchestration_v2/test_cache_control.py index 786d1845..c9b000b8 100644 --- a/packages/gen/integration_tests/orchestration_v2/test_cache_control.py +++ b/packages/gen/integration_tests/orchestration_v2/test_cache_control.py @@ -1,21 +1,12 @@ """ -Unit and integration tests for prompt caching (cache_control) via Orchestration V2. +Live integration tests for prompt caching (cache_control) via Orchestration V2. Caching is supported for Anthropic Claude and Amazon Nova models. -The test targets anthropic--claude-4.6-sonnet (1024-token minimum, 5m and 1h TTLs). +The tests target anthropic--claude-4.6-sonnet (1024-token minimum, 5m and 1h TTLs). Wire path: ai-sdk-python -> SAP AI Core /v2/completion -> SAP LiteLLM fork -> Anthropic API -cache_control is serialized as a plain JSON key on the content block or tool dict by -Pydantic model_dump(). The SAP LiteLLM fork translates it into Anthropic's native -prompt-caching format. - -The spec defines cache_control on three schema-level attachment points: - - TextContent.cache_control (TextPart in py) - - UserChatMessageContentItem.cache_control (TextPart / ImagePart in py) - - ChatCompletionTool.cache_control - Response fields (from SAP AI Core orchestration docs): prompt_tokens_details.cached_tokens -- tokens read from cache (hit) prompt_tokens_details.cache_creation_tokens -- tokens written to cache (miss) @@ -28,9 +19,8 @@ from gen_ai_hub.orchestration_v2.models.config import OrchestrationConfig, ModuleConfig from gen_ai_hub.orchestration_v2.models.llm_model_details import LLMModelDetails from gen_ai_hub.orchestration_v2.models.message import SystemMessage, UserMessage -from gen_ai_hub.orchestration_v2.models.multimodal_items import TextPart, ImagePart, ImageUrl +from gen_ai_hub.orchestration_v2.models.multimodal_items import TextPart from gen_ai_hub.orchestration_v2.models.template import Template, PromptTemplatingModuleConfig -from gen_ai_hub.orchestration_v2.models.tools import ChatCompletionTool, FunctionTool, FunctionObject from gen_ai_hub.orchestration_v2.service import OrchestrationService from integration_tests.orchestration_v2.test_base import OrchestrationServiceTestBase from integration_tests.test_helpers import retry_on_429_or_503 @@ -84,125 +74,6 @@ def _config(messages, tools=None): ) -class TestCacheControlSerialization(unittest.TestCase): - """Unit tests: verify cache_control serializes correctly without a network call.""" - - # ------------------------------------------------------------------ - # CacheControl model - # ------------------------------------------------------------------ - - def test_cache_control_default_ttl_omits_key(self): - """CacheControl() with no TTL serializes to {"type": "ephemeral"}.""" - d = CacheControl().model_dump(by_alias=True) - self.assertEqual(d, {"type": "ephemeral"}) - self.assertNotIn("ttl", d) - - def test_cache_control_5m_ttl(self): - """CacheControl(ttl="5m") serializes with ttl field.""" - d = CacheControl(ttl="5m").model_dump(by_alias=True) - self.assertEqual(d, {"type": "ephemeral", "ttl": "5m"}) - - def test_cache_control_1h_ttl(self): - """CacheControl(ttl="1h") serializes with ttl field.""" - d = CacheControl(ttl="1h").model_dump(by_alias=True) - self.assertEqual(d, {"type": "ephemeral", "ttl": "1h"}) - - # ------------------------------------------------------------------ - # TextPart with cache_control - # ------------------------------------------------------------------ - - def test_text_part_with_cache_control(self): - """TextPart with cache_control serializes the cache_control block.""" - part = TextPart(text="hello", cache_control=CacheControl()) - d = part.model_dump(by_alias=True, exclude_none=True) - self.assertEqual(d["type"], "text") - self.assertEqual(d["text"], "hello") - self.assertEqual(d["cache_control"], {"type": "ephemeral"}) - - def test_text_part_without_cache_control_omits_key(self): - """TextPart without cache_control does not emit the key.""" - part = TextPart(text="hello") - d = part.model_dump(by_alias=True, exclude_none=True) - self.assertNotIn("cache_control", d) - - def test_text_part_with_1h_ttl(self): - """TextPart with CacheControl(ttl='1h') serializes the ttl field.""" - part = TextPart(text="hello", cache_control=CacheControl(ttl="1h")) - d = part.model_dump(by_alias=True, exclude_none=True) - self.assertEqual(d["cache_control"], {"type": "ephemeral", "ttl": "1h"}) - - # ------------------------------------------------------------------ - # ImagePart with cache_control - # ------------------------------------------------------------------ - - def test_image_part_with_cache_control(self): - """ImagePart with cache_control serializes the cache_control block.""" - part = ImagePart( - image_url=ImageUrl(url="https://example.com/img.png"), - cache_control=CacheControl(), - ) - d = part.model_dump(by_alias=True, exclude_none=True) - self.assertEqual(d["type"], "image_url") - self.assertEqual(d["cache_control"], {"type": "ephemeral"}) - - def test_image_part_without_cache_control_omits_key(self): - """ImagePart without cache_control does not emit the key.""" - part = ImagePart(image_url=ImageUrl(url="https://example.com/img.png")) - d = part.model_dump(by_alias=True, exclude_none=True) - self.assertNotIn("cache_control", d) - - def test_image_part_with_1h_ttl(self): - """ImagePart with CacheControl(ttl='1h') serializes the ttl field.""" - part = ImagePart( - image_url=ImageUrl(url="https://example.com/img.png"), - cache_control=CacheControl(ttl="1h"), - ) - d = part.model_dump(by_alias=True, exclude_none=True) - self.assertEqual(d["cache_control"], {"type": "ephemeral", "ttl": "1h"}) - - # ------------------------------------------------------------------ - # ChatCompletionTool with cache_control - # ------------------------------------------------------------------ - - def test_tool_cache_control_serialized(self): - """cache_control on a ChatCompletionTool appears at the tool level.""" - tool = FunctionTool( - function=FunctionObject( - name="classify", - description="Classify input.", - parameters={"type": "object", "properties": {}}, - ), - cache_control=CacheControl(), - ) - d = tool.model_dump(by_alias=True, exclude_none=True) - self.assertEqual(d["cache_control"], {"type": "ephemeral"}) - - def test_tool_without_cache_control_omits_key(self): - """A tool without cache_control does not emit the key.""" - tool = FunctionTool( - function=FunctionObject( - name="classify", - description="Classify input.", - parameters={"type": "object", "properties": {}}, - ), - ) - d = tool.model_dump(by_alias=True, exclude_none=True) - self.assertNotIn("cache_control", d) - - def test_tool_with_1h_ttl(self): - """cache_control with ttl='1h' on a tool serializes the ttl field.""" - tool = FunctionTool( - function=FunctionObject( - name="classify", - description="Classify input.", - parameters={"type": "object", "properties": {}}, - ), - cache_control=CacheControl(ttl="1h"), - ) - d = tool.model_dump(by_alias=True, exclude_none=True) - self.assertEqual(d["cache_control"], {"type": "ephemeral", "ttl": "1h"}) - - class TestPromptCachingLive(OrchestrationServiceTestBase): """Live integration tests against the SAP AI Core orchestration V2 service.""" diff --git a/packages/gen/tests/orchestration_v2/test_cache_control_v2.py b/packages/gen/tests/orchestration_v2/test_cache_control_v2.py new file mode 100644 index 00000000..70830eb0 --- /dev/null +++ b/packages/gen/tests/orchestration_v2/test_cache_control_v2.py @@ -0,0 +1,117 @@ +""" +Unit tests for prompt-caching serialization (cache_control) in Orchestration V2. + +Covers the three spec attachment points: + - TextContent.cache_control (TextPart) + - UserChatMessageContentItem (TextPart / ImagePart) + - ChatCompletionTool.cache_control +""" +import unittest + +from gen_ai_hub.orchestration_v2.models.cache_control import CacheControl +from gen_ai_hub.orchestration_v2.models.multimodal_items import TextPart, ImagePart, ImageUrl +from gen_ai_hub.orchestration_v2.models.tools import FunctionTool, FunctionObject + + +class TestCacheControlModel(unittest.TestCase): + """CacheControl serialization.""" + + def test_default_ttl_omits_key(self): + """CacheControl() with no TTL serializes to {"type": "ephemeral"}.""" + d = CacheControl().model_dump(by_alias=True) + self.assertEqual(d, {"type": "ephemeral"}) + self.assertNotIn("ttl", d) + + def test_5m_ttl(self): + d = CacheControl(ttl="5m").model_dump(by_alias=True) + self.assertEqual(d, {"type": "ephemeral", "ttl": "5m"}) + + def test_1h_ttl(self): + d = CacheControl(ttl="1h").model_dump(by_alias=True) + self.assertEqual(d, {"type": "ephemeral", "ttl": "1h"}) + + +class TestTextPartCacheControl(unittest.TestCase): + """TextPart.cache_control serialization.""" + + def test_with_cache_control(self): + part = TextPart(text="hello", cache_control=CacheControl()) + d = part.model_dump(by_alias=True, exclude_none=True) + self.assertEqual(d["type"], "text") + self.assertEqual(d["text"], "hello") + self.assertEqual(d["cache_control"], {"type": "ephemeral"}) + + def test_without_cache_control_omits_key(self): + part = TextPart(text="hello") + d = part.model_dump(by_alias=True, exclude_none=True) + self.assertNotIn("cache_control", d) + + def test_1h_ttl(self): + part = TextPart(text="hello", cache_control=CacheControl(ttl="1h")) + d = part.model_dump(by_alias=True, exclude_none=True) + self.assertEqual(d["cache_control"], {"type": "ephemeral", "ttl": "1h"}) + + +class TestImagePartCacheControl(unittest.TestCase): + """ImagePart.cache_control serialization.""" + + def _image_part(self, **kwargs): + return ImagePart(image_url=ImageUrl(url="https://example.com/img.png"), **kwargs) + + def test_with_cache_control(self): + d = self._image_part(cache_control=CacheControl()).model_dump(by_alias=True, exclude_none=True) + self.assertEqual(d["type"], "image_url") + self.assertEqual(d["cache_control"], {"type": "ephemeral"}) + + def test_without_cache_control_omits_key(self): + d = self._image_part().model_dump(by_alias=True, exclude_none=True) + self.assertNotIn("cache_control", d) + + def test_1h_ttl(self): + d = self._image_part(cache_control=CacheControl(ttl="1h")).model_dump(by_alias=True, exclude_none=True) + self.assertEqual(d["cache_control"], {"type": "ephemeral", "ttl": "1h"}) + + +class TestFunctionToolCacheControl(unittest.TestCase): + """ChatCompletionTool.cache_control serialization via FunctionTool.""" + + def _tool(self, **kwargs): + return FunctionTool( + function=FunctionObject( + name="classify", + description="Classify input.", + parameters={"type": "object", "properties": {}}, + ), + **kwargs, + ) + + def test_with_cache_control(self): + d = self._tool(cache_control=CacheControl()).model_dump(by_alias=True, exclude_none=True) + self.assertEqual(d["cache_control"], {"type": "ephemeral"}) + + def test_without_cache_control_omits_key(self): + d = self._tool().model_dump(by_alias=True, exclude_none=True) + self.assertNotIn("cache_control", d) + + def test_1h_ttl(self): + d = self._tool(cache_control=CacheControl(ttl="1h")).model_dump(by_alias=True, exclude_none=True) + self.assertEqual(d["cache_control"], {"type": "ephemeral", "ttl": "1h"}) + + def test_type_field_serializes(self): + """type_ with alias 'type' must appear in output.""" + d = self._tool().model_dump(by_alias=True, exclude_none=True) + self.assertEqual(d["type"], "function") + + def test_no_duplicate_type_from_subclass(self): + """FunctionTool must not declare its own type_ field (inherits from ChatCompletionTool).""" + import inspect + own_fields = FunctionTool.model_fields + # 'type_' is defined on ChatCompletionTool; FunctionTool should only add 'function' + self.assertIn("function", own_fields) + # Ensure serialization is still correct (regression guard) + d = self._tool().model_dump(by_alias=True) + self.assertEqual(d["type"], "function") + + +if __name__ == "__main__": + unittest.main() From 39d9000d043e7aa67d9f0804851a47fac0598d77 Mon Sep 17 00:00:00 2001 From: Yamac Ay Date: Thu, 10 Sep 2026 18:19:53 +0200 Subject: [PATCH 04/11] docstring fixed --- .../gen/gen_ai_hub/orchestration_v2/models/tools.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py b/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py index b8ef1979..4ac6be73 100644 --- a/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py +++ b/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py @@ -96,8 +96,17 @@ class FunctionObject(BaseModel): class FunctionTool(ChatCompletionTool): """A callable function tool for OpenAI-style function calling. + Inherits all fields from :class:`ChatCompletionTool`: + Args: - function: The function definition (name, description, parameters). + type: Always ``"function"``. Serialized via the ``type`` alias. + cache_control: Prompt-caching directive. Supported on Anthropic Claude only; + not supported on Amazon Nova. + + Additional args: + + function: The function definition — name, description, parameters schema, + and optional strict flag. See :class:`FunctionObject`. """ function: FunctionObject From e0a040be710cc9cd76a93604b0cfd538aaf1f170 Mon Sep 17 00:00:00 2001 From: Yamac Ay Date: Fri, 11 Sep 2026 15:29:27 +0200 Subject: [PATCH 05/11] remove default ephemeral, fix docstrings, consistent quoting --- .../orchestration_v2/models/cache_control.py | 18 ++++------------- .../orchestration_v2/test_cache_control.py | 8 ++++---- .../orchestration_v2/test_cache_control_v2.py | 20 +++++++++---------- 3 files changed, 18 insertions(+), 28 deletions(-) diff --git a/packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py b/packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py index 348d7d95..1d9f8a90 100644 --- a/packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py +++ b/packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py @@ -18,25 +18,15 @@ class CacheControl(BaseModel): - Amazon Nova: system and user content blocks only (no tools, no TTL). Attach ``CacheControl`` directly to a content block (``TextPart``, ``ImagePart``) or - to a ``ChatCompletionTool``. For the "last-block shorthand" pass ``cache_control`` to - ``OrchestrationService.run()``; it calls ``apply_cache_control_to_last_message()`` - automatically. - + to a ``ChatCompletionTool``. + Args: - type: Always ``"ephemeral"``. Only value supported by the API. + type: ``"ephemeral"`` ttl: Cache duration. ``"5m"`` (default) or ``"1h"`` (select Anthropic models only). Omit for Amazon Nova or when the default is sufficient. - - Example:: - - from gen_ai_hub.orchestration_v2.models.cache_control import CacheControl - from gen_ai_hub.orchestration_v2.models.multimodal_items import TextPart - - block = TextPart(text="Long context...", cache_control=CacheControl()) - block_1h = TextPart(text="Long context...", cache_control=CacheControl(ttl="1h")) """ - type: Literal["ephemeral"] = "ephemeral" + type: Literal["ephemeral"] ttl: Optional[Literal["5m", "1h"]] = None @model_serializer(mode="wrap") diff --git a/packages/gen/integration_tests/orchestration_v2/test_cache_control.py b/packages/gen/integration_tests/orchestration_v2/test_cache_control.py index c9b000b8..c6ff8eeb 100644 --- a/packages/gen/integration_tests/orchestration_v2/test_cache_control.py +++ b/packages/gen/integration_tests/orchestration_v2/test_cache_control.py @@ -95,7 +95,7 @@ def test_cache_miss_on_first_call(self): cache_control breakpoint was accepted by the server. """ config = _config([ - SystemMessage(content=[TextPart(text=_LONG_SYSTEM_PROMPT, cache_control=CacheControl())]), + SystemMessage(content=[TextPart(text=_LONG_SYSTEM_PROMPT, cache_control=CacheControl(type="ephemeral"))]), UserMessage(content="In one word: what language did Romans speak?"), ]) response = self.service.run(config=config) @@ -116,7 +116,7 @@ def test_cache_miss_on_first_call(self): def test_cache_hit_on_repeated_call(self): """Second call with the same cache breakpoint produces cached_tokens > 0.""" config = _config([ - SystemMessage(content=[TextPart(text=_LONG_SYSTEM_PROMPT, cache_control=CacheControl())]), + SystemMessage(content=[TextPart(text=_LONG_SYSTEM_PROMPT, cache_control=CacheControl(type="ephemeral"))]), UserMessage(content="In one word: what language did Romans speak?"), ]) self.service.run(config=config) # populate cache @@ -134,10 +134,10 @@ def test_cache_hit_on_repeated_call(self): @retry_on_429_or_503(max_retries=3, initial_delay=2.0, backoff_factor=2.0) def test_explicit_ttl_1h_via_text_part(self): - """Attaching CacheControl(ttl='1h') directly to a TextPart returns + """Attaching CacheControl(type='ephemeral', ttl='1h') directly to a TextPart returns cache_creation_token_details with ephemeral_1h_input_tokens.""" config = _config([ - SystemMessage(content=[TextPart(text=_LONG_SYSTEM_PROMPT, cache_control=CacheControl(ttl="1h"))]), + SystemMessage(content=[TextPart(text=_LONG_SYSTEM_PROMPT, cache_control=CacheControl(type='ephemeral', ttl='1h'))]), UserMessage(content="Name the last Western Roman emperor."), ]) response = self.service.run(config=config) diff --git a/packages/gen/tests/orchestration_v2/test_cache_control_v2.py b/packages/gen/tests/orchestration_v2/test_cache_control_v2.py index 70830eb0..adadf266 100644 --- a/packages/gen/tests/orchestration_v2/test_cache_control_v2.py +++ b/packages/gen/tests/orchestration_v2/test_cache_control_v2.py @@ -17,17 +17,17 @@ class TestCacheControlModel(unittest.TestCase): """CacheControl serialization.""" def test_default_ttl_omits_key(self): - """CacheControl() with no TTL serializes to {"type": "ephemeral"}.""" - d = CacheControl().model_dump(by_alias=True) + """CacheControl(type='ephemeral') with no TTL serializes to {"type": "ephemeral"}.""" + d = CacheControl(type="ephemeral").model_dump(by_alias=True) self.assertEqual(d, {"type": "ephemeral"}) self.assertNotIn("ttl", d) def test_5m_ttl(self): - d = CacheControl(ttl="5m").model_dump(by_alias=True) + d = CacheControl(type="ephemeral", ttl="5m").model_dump(by_alias=True) self.assertEqual(d, {"type": "ephemeral", "ttl": "5m"}) def test_1h_ttl(self): - d = CacheControl(ttl="1h").model_dump(by_alias=True) + d = CacheControl(type="ephemeral", ttl="1h").model_dump(by_alias=True) self.assertEqual(d, {"type": "ephemeral", "ttl": "1h"}) @@ -35,7 +35,7 @@ class TestTextPartCacheControl(unittest.TestCase): """TextPart.cache_control serialization.""" def test_with_cache_control(self): - part = TextPart(text="hello", cache_control=CacheControl()) + part = TextPart(text="hello", cache_control=CacheControl(type="ephemeral")) d = part.model_dump(by_alias=True, exclude_none=True) self.assertEqual(d["type"], "text") self.assertEqual(d["text"], "hello") @@ -47,7 +47,7 @@ def test_without_cache_control_omits_key(self): self.assertNotIn("cache_control", d) def test_1h_ttl(self): - part = TextPart(text="hello", cache_control=CacheControl(ttl="1h")) + part = TextPart(text="hello", cache_control=CacheControl(type="ephemeral", ttl="1h")) d = part.model_dump(by_alias=True, exclude_none=True) self.assertEqual(d["cache_control"], {"type": "ephemeral", "ttl": "1h"}) @@ -59,7 +59,7 @@ def _image_part(self, **kwargs): return ImagePart(image_url=ImageUrl(url="https://example.com/img.png"), **kwargs) def test_with_cache_control(self): - d = self._image_part(cache_control=CacheControl()).model_dump(by_alias=True, exclude_none=True) + d = self._image_part(cache_control=CacheControl(type="ephemeral")).model_dump(by_alias=True, exclude_none=True) self.assertEqual(d["type"], "image_url") self.assertEqual(d["cache_control"], {"type": "ephemeral"}) @@ -68,7 +68,7 @@ def test_without_cache_control_omits_key(self): self.assertNotIn("cache_control", d) def test_1h_ttl(self): - d = self._image_part(cache_control=CacheControl(ttl="1h")).model_dump(by_alias=True, exclude_none=True) + d = self._image_part(cache_control=CacheControl(type="ephemeral", ttl="1h")).model_dump(by_alias=True, exclude_none=True) self.assertEqual(d["cache_control"], {"type": "ephemeral", "ttl": "1h"}) @@ -86,7 +86,7 @@ def _tool(self, **kwargs): ) def test_with_cache_control(self): - d = self._tool(cache_control=CacheControl()).model_dump(by_alias=True, exclude_none=True) + d = self._tool(cache_control=CacheControl(type="ephemeral")).model_dump(by_alias=True, exclude_none=True) self.assertEqual(d["cache_control"], {"type": "ephemeral"}) def test_without_cache_control_omits_key(self): @@ -94,7 +94,7 @@ def test_without_cache_control_omits_key(self): self.assertNotIn("cache_control", d) def test_1h_ttl(self): - d = self._tool(cache_control=CacheControl(ttl="1h")).model_dump(by_alias=True, exclude_none=True) + d = self._tool(cache_control=CacheControl(type="ephemeral", ttl="1h")).model_dump(by_alias=True, exclude_none=True) self.assertEqual(d["cache_control"], {"type": "ephemeral", "ttl": "1h"}) def test_type_field_serializes(self): From 48f4a6c28e9a48c3df2f23333d5ba94bbc1064c4 Mon Sep 17 00:00:00 2001 From: Yamac Ay Date: Tue, 15 Sep 2026 15:14:37 +0200 Subject: [PATCH 06/11] fix: docstring fixes and Field(exclude=False) to None --- .../gen_ai_hub/orchestration_v2/models/cache_control.py | 6 +----- .../gen_ai_hub/orchestration_v2/models/multimodal_items.py | 6 ++---- packages/gen/gen_ai_hub/orchestration_v2/models/tools.py | 7 +++---- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py b/packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py index 1d9f8a90..2d232c3f 100644 --- a/packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py +++ b/packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py @@ -1,4 +1,4 @@ -"""Cache control for prompt caching on supported Anthropic and Amazon Nova models.""" +"""Cache control for prompt caching on supported models.""" from typing import Any, Dict, Literal, Optional from pydantic import model_serializer @@ -13,10 +13,6 @@ class CacheControl(BaseModel): results for that content and reuses them on subsequent requests within the TTL window, reducing both latency and token costs. - Supported models: - - Anthropic Claude: system and user content blocks; tools. - - Amazon Nova: system and user content blocks only (no tools, no TTL). - Attach ``CacheControl`` directly to a content block (``TextPart``, ``ImagePart``) or to a ``ChatCompletionTool``. diff --git a/packages/gen/gen_ai_hub/orchestration_v2/models/multimodal_items.py b/packages/gen/gen_ai_hub/orchestration_v2/models/multimodal_items.py index 9f17c893..2f46ed71 100644 --- a/packages/gen/gen_ai_hub/orchestration_v2/models/multimodal_items.py +++ b/packages/gen/gen_ai_hub/orchestration_v2/models/multimodal_items.py @@ -41,11 +41,10 @@ class TextPart(BaseModel): type: The type identifier, defaulting to "text". cache_control: Optional cache control settings for prompt caching. - Only supported for Anthropic Claude and Amazon Nova models. """ text: str type_: Literal["text"] = Field(default="text", alias="type") - cache_control: Optional[CacheControl] = Field(default=None, exclude=False) + cache_control: Optional[CacheControl] = None class ImageUrl(BaseModel): @@ -71,11 +70,10 @@ class ImagePart(BaseModel): type: The type identifier, defaulting to "image_url". cache_control: Optional cache control settings for prompt caching. - Only supported for Anthropic Claude models. """ image_url: ImageUrl type_: Literal["image_url"] = Field(default="image_url", alias="type") - cache_control: Optional[CacheControl] = Field(default=None, exclude=False) + cache_control: Optional[CacheControl] = None ContentPart = Union[TextPart, ImagePart] diff --git a/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py b/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py index 4ac6be73..9177b02a 100644 --- a/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py +++ b/packages/gen/gen_ai_hub/orchestration_v2/models/tools.py @@ -70,8 +70,8 @@ class ChatCompletionTool(BaseModel): """A tool the model may call, identified by type ``"function"``. Args: - cache_control: Prompt-caching directive. Supported on Anthropic Claude only; - not supported on Amazon Nova. + type (Literal["function"]): The type of the tool. Currently, only function is supported. + cache_control: Prompt-caching directive. """ type_: Literal["function"] = Field(default="function", alias="type") cache_control: Optional[CacheControl] = None @@ -100,8 +100,7 @@ class FunctionTool(ChatCompletionTool): Args: type: Always ``"function"``. Serialized via the ``type`` alias. - cache_control: Prompt-caching directive. Supported on Anthropic Claude only; - not supported on Amazon Nova. + cache_control: Prompt-caching directive. Additional args: From f144e3f8c7e9b9b4dae97309eea1b3067820b7e7 Mon Sep 17 00:00:00 2001 From: Yamac Ay Date: Wed, 16 Sep 2026 10:10:00 +0200 Subject: [PATCH 07/11] fix docstring --- .../gen/gen_ai_hub/orchestration_v2/models/cache_control.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py b/packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py index 2d232c3f..affd4e19 100644 --- a/packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py +++ b/packages/gen/gen_ai_hub/orchestration_v2/models/cache_control.py @@ -18,8 +18,7 @@ class CacheControl(BaseModel): Args: type: ``"ephemeral"`` - ttl: Cache duration. ``"5m"`` (default) or ``"1h"`` (select Anthropic - models only). Omit for Amazon Nova or when the default is sufficient. + ttl: Cache duration. ``"5m"`` (default) or ``"1h"``. """ type: Literal["ephemeral"] From 1daf0b611243bc57e9ea11726c96b5e15c4df558 Mon Sep 17 00:00:00 2001 From: Yamac Ay Date: Wed, 16 Sep 2026 11:33:51 +0200 Subject: [PATCH 08/11] update examples --- .../examples/orchestration-service2.ipynb | 1013 ++++++++++------- 1 file changed, 590 insertions(+), 423 deletions(-) diff --git a/packages/gen/docs/gen_ai_hub/examples/orchestration-service2.ipynb b/packages/gen/docs/gen_ai_hub/examples/orchestration-service2.ipynb index 6ade4d46..a34a2ae2 100644 --- a/packages/gen/docs/gen_ai_hub/examples/orchestration-service2.ipynb +++ b/packages/gen/docs/gen_ai_hub/examples/orchestration-service2.ipynb @@ -1,72 +1,77 @@ { "cells": [ { - "metadata": {}, "cell_type": "markdown", + "id": "d8b115a4d92fd4db", + "metadata": {}, "source": [ "(orchestration2)=\n", "# Orchestration Service V2 API" - ], - "id": "d8b115a4d92fd4db" + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "This notebook demonstrates how to use the SDK to interact with the Orchestration Service V2, enabling the creation of AI-driven workflows by seamlessly integrating various modules, such as templating, large language models (LLMs), data masking and content filtering. By leveraging these modules, you can build complex, automated workflows that enhance the capabilities of your AI solutions. For more details on configuring and using these modules, please refer to the [Orchestration Service Documentation](https://help.sap.com/docs/ai-launchpad/sap-ai-launchpad/orchestration).", - "id": "25e1a6ae2c503530" + "id": "25e1a6ae2c503530", + "metadata": {}, + "source": [ + "This notebook demonstrates how to use the SDK to interact with the Orchestration Service V2, enabling the creation of AI-driven workflows by seamlessly integrating various modules, such as templating, large language models (LLMs), data masking and content filtering. By leveraging these modules, you can build complex, automated workflows that enhance the capabilities of your AI solutions. For more details on configuring and using these modules, please refer to the [Orchestration Service Documentation](https://help.sap.com/docs/ai-launchpad/sap-ai-launchpad/orchestration)." + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "ef9eb9a6a7b70d21", + "metadata": {}, "source": [ "## Prerequisite\n", "\n", "> **Important:** Before you begin using the SDK, make sure to set up a virtual deployment of the Orchestration Service.\n", "\n", "For detailed guidance on setting up the Orchestration Service, please refer to the setup guide [here](https://help.sap.com/docs/ai-launchpad/sap-ai-launchpad/create-deployment-for-orchestration)." - ], - "id": "ef9eb9a6a7b70d21" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "5b5932e69ddd9d9b", + "metadata": {}, "source": [ "## Authentication\n", "\n", "By default, the `OrchestrationService` initializes a `GenAIHubProxyClient`, which automatically configures credentials using configuration files or environment variables, as outlined in the *Introduction* section.\n", "\n", "If you prefer to set credentials manually, you can provide a custom instance using the `proxy_client` parameter." - ], - "id": "5b5932e69ddd9d9b" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "66a8b36fcc724b6c", + "metadata": {}, "source": [ "## Basic Orchestration Pipeline\n", "\n", "Let's walk through a basic orchestration pipeline for a translation task." - ], - "id": "66a8b36fcc724b6c" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "efa18d8a1bc94fa5", + "metadata": {}, "source": [ "### Step 1: Define the Template and Default Input Values\n", "\n", "The `Template` class is used to define structured message templates for generating dynamic interactions with language models. In this example, the template is designed for a translation assistant, allowing users to specify a language and text for translation." - ], - "id": "efa18d8a1bc94fa5" + ] }, { + "cell_type": "code", + "execution_count": 1, + "id": "68ae8781e1f37767", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:21:27.519791Z", "start_time": "2026-03-19T07:21:26.902024Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import Template, SystemMessage, UserMessage\n", "\n", @@ -77,70 +82,74 @@ " ],\n", " defaults={\"to_lang\": \"German\"}\n", " )" - ], - "id": "68ae8781e1f37767", - "outputs": [], - "execution_count": 1 + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "This template can be used to create translation requests where the language and text to be translated are specified dynamically. The placeholders in the `UserMessage` will be replaced with the actual values provided at runtime, and the default value for the language is set to German.", - "id": "8aa1bf41dd94da85" + "id": "8aa1bf41dd94da85", + "metadata": {}, + "source": [ + "This template can be used to create translation requests where the language and text to be translated are specified dynamically. The placeholders in the `UserMessage` will be replaced with the actual values provided at runtime, and the default value for the language is set to German." + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "701646ebfe806c03", + "metadata": {}, "source": [ "### Step 2: Define the LLM\n", "\n", "The `LLM` class is used to configure and initialize a language model for generating text based on specific parameters. In this example, we'll use the `gpt-4o` model to perform the translation task.\n", "\n", "**Note:** The Orchestration Service automatically manages the virtual deployment of the language model, so no additional setup is needed on your end." - ], - "id": "701646ebfe806c03" + ] }, { + "cell_type": "code", + "execution_count": 2, + "id": "55c03da66cf6e8de", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:21:35.847306Z", "start_time": "2026-03-19T07:21:35.845208Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import LLMModelDetails\n", "\n", "llm = LLMModelDetails(name=\"gpt-5-nano\", params={\"max_completion_tokens\": 512})" - ], - "id": "55c03da66cf6e8de", - "outputs": [], - "execution_count": 2 + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "Initializes the language model to use the `gpt-5-nano` model. It will generate responses up to 512 tokens in length.", - "id": "82ae689dce681b5f" + "id": "82ae689dce681b5f", + "metadata": {}, + "source": [ + "Initializes the language model to use the `gpt-5-nano` model. It will generate responses up to 512 tokens in length." + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "dc3f2a71cce77aa", + "metadata": {}, "source": [ "### Step 3: Create the Orchestration Configuration\n", "\n", "The `OrchestrationConfig` class defines a configuration for integrating various modules, such as templates and language models, into a cohesive orchestration setup. It specifies how these components interact and are configured to achieve the desired operational scenario." - ], - "id": "dc3f2a71cce77aa" + ] }, { + "cell_type": "code", + "execution_count": 3, + "id": "62b4386a1a48c7b5", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:21:52.276460Z", "start_time": "2026-03-19T07:21:52.271917Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import PromptTemplatingModuleConfig, ModuleConfig, OrchestrationConfig\n", "\n", @@ -150,86 +159,86 @@ "module_config = ModuleConfig(prompt_templating=prompt_template)\n", "\n", "config = OrchestrationConfig(modules=module_config)" - ], - "id": "62b4386a1a48c7b5", - "outputs": [], - "execution_count": 3 + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "10f94bed8b6ef93c", + "metadata": {}, "source": [ "### Step 4: Run the Orchestration Request\n", "\n", "The `OrchestrationService` class is used to interact with a orchestration service instance by providing configuration details to initiate and manage its operations." - ], - "id": "10f94bed8b6ef93c" + ] }, { + "cell_type": "code", + "execution_count": 4, + "id": "427b94b07e2ba67f", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:22:00.455920Z", "start_time": "2026-03-19T07:21:59.415073Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import OrchestrationService\n", "\n", "orchestration_service = OrchestrationService(config=config)" - ], - "id": "427b94b07e2ba67f", - "outputs": [], - "execution_count": 4 + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "Call the `run` method with the required `placeholder values`. The service will process the input according to the configuration and return the result.", - "id": "3151fa9176fb585d" + "id": "3151fa9176fb585d", + "metadata": {}, + "source": [ + "Call the `run` method with the required `placeholder values`. The service will process the input according to the configuration and return the result." + ] }, { - "metadata": {}, "cell_type": "code", + "execution_count": null, + "id": "3684a72a3856ca8f", + "metadata": {}, + "outputs": [], "source": [ "result = orchestration_service.run(placeholder_values={\"user_query\": \"The Orchestration Service is working!\"})\n", "print(result.final_result.choices[0].message.content)" - ], - "id": "3684a72a3856ca8f", - "outputs": [], - "execution_count": null + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "1bd44dc91b57793d", + "metadata": {}, "source": [ "(prompt_registry)=\n", "#### Referencing Templates in the Prompt Registry\n", " In Step 3 you can also use a prompt template reference, which allows you to reuse existing templates stored in the Prompt Registry." - ], - "id": "1bd44dc91b57793d" + ] }, { + "cell_type": "code", + "execution_count": 6, + "id": "118cf8c87dcfcf80", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:22:21.413392Z", "start_time": "2026-03-19T07:22:21.411795Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import TemplateRefByID, TemplateRefByScenarioNameVersion\n", "\n", "template_by_id = TemplateRefByID(id=\"648871d9-b207-441c-8c13-afee71b0dbec\") # this is just an example id\n", "template_by_names = TemplateRefByScenarioNameVersion(scenario=\"translation\", name=\"translate_text\", version=\"0.1.0\")" - ], - "id": "118cf8c87dcfcf80", - "outputs": [], - "execution_count": 6 + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "8af833056514e215", + "metadata": {}, "source": [ "(response_format)=\n", "#### Overview of response_format Parameter Options\n", @@ -241,17 +250,19 @@ "2. **json_object**: Under this setting, the model's output is structured as a JSON object. This is useful for applications that handle data in JSON format, enabling easy integration with web applications and APIs.\n", "\n", "3. **json_schema**: This setting allows the model's output to adhere to a defined JSON schema. This is particularly useful for applications that require strict data validation, ensuring the output matches a predefined schema." - ], - "id": "8af833056514e215" + ] }, { + "cell_type": "code", + "execution_count": 7, + "id": "6344a28c59c46dbe", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:22:39.198453Z", "start_time": "2026-03-19T07:22:39.196095Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import SystemMessage, UserMessage, Template, ResponseFormatText\n", "\n", @@ -266,19 +277,19 @@ "\n", "# Response:\n", "# The first man on the moon was Neil Armstrong." - ], - "id": "6344a28c59c46dbe", - "outputs": [], - "execution_count": 7 + ] }, { + "cell_type": "code", + "execution_count": 8, + "id": "b07c3c0b320e5026", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:22:54.254255Z", "start_time": "2026-03-19T07:22:54.246206Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import SystemMessage, UserMessage, Template, ResponseFormatJsonObject\n", "\n", @@ -295,25 +306,27 @@ "# {\n", "# \"First_man_on_the_moon\": \"Neil Armstrong\"\n", "# }" - ], - "id": "b07c3c0b320e5026", - "outputs": [], - "execution_count": 8 + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "**Important:** When using `response_format` as json_object, ensure that messages contain the word 'json' in some form.", - "id": "3bff15b49f9cbef7" + "id": "3bff15b49f9cbef7", + "metadata": {}, + "source": [ + "**Important:** When using `response_format` as json_object, ensure that messages contain the word 'json' in some form." + ] }, { + "cell_type": "code", + "execution_count": 9, + "id": "4779aa6b1627fd8c", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:23:14.596775Z", "start_time": "2026-03-19T07:23:14.585103Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import SystemMessage, UserMessage, Template, ResponseFormatJsonSchema, JSONResponseSchema\n", "\n", @@ -349,35 +362,33 @@ "# \"firstName\": \"Neil\",\n", "# \"lastName\": \"Armstrong\"\n", "# }" - ], - "id": "4779aa6b1627fd8c", - "outputs": [], - "execution_count": 9 + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "af41bbe9b0279faf", + "metadata": {}, "source": [ "(orchestration_deployment)=\n", "## Understanding Deployment Resolution\n", "\n", "The `OrchestrationService` class provides multiple ways to specify and target orchestration deployments when sending requests. Below are the available options:" - ], - "id": "af41bbe9b0279faf" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "9ae1192eb2ef5efd", + "metadata": {}, "source": [ "### Default Behavior\n", "\n", "If no parameters are provided, the `OrchestrationService` automatically searches for a `RUNNING` deployment. If multiple running deployments exist, the service selects the most recently created one." - ], - "id": "9ae1192eb2ef5efd" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "66c869b15585442a", + "metadata": {}, "source": [ "### Direct Deployment Specification\n", "\n", @@ -389,12 +400,12 @@ "\n", "2. **Deployment ID** (`deployment_id`):\n", " - Use the unique identifier assigned to the deployment instead of the URL." - ], - "id": "66c869b15585442a" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "e65b02881577838d", + "metadata": {}, "source": [ "### Config-Based Specification\n", "\n", @@ -407,18 +418,20 @@ " - The service looks for a `RUNNING` deployment that matches the specified configuration name.\n", "\n", "If multiple deployments match the given configuration criteria, the most recently created one will be selected automatically." - ], - "id": "e65b02881577838d" + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "## Optional Modules", - "id": "ee0538ff9e3de88f" + "id": "ee0538ff9e3de88f", + "metadata": {}, + "source": [ + "## Optional Modules" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "a102e0350b4fd2b2", + "metadata": {}, "source": [ "### Data Masking\n", "\n", @@ -438,17 +451,19 @@ "- **entities**: Specify which types of entities to mask (e.g., EMAIL, PHONE, PERSON).\n", "- **allowlist**: Provide specific terms or patterns that should be excluded from masking, even if they match entity types.\n", "- **mask_grounding_input**: When enabled, ensures that masking is also applied to the context provided to the grounding module." - ], - "id": "a102e0350b4fd2b2" + ] }, { + "cell_type": "code", + "execution_count": 10, + "id": "e5f818a811909e83", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:24:32.758004Z", "start_time": "2026-03-19T07:24:28.674275Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2.utils import load_text_file\n", "from gen_ai_hub.orchestration_v2 import (SystemMessage, UserMessage, Template, PromptTemplatingModuleConfig,\n", @@ -494,38 +509,40 @@ " config=config,\n", " placeholder_values={\"orgCV\": cv_as_string}\n", ")" - ], - "id": "e5f818a811909e83", - "outputs": [], - "execution_count": 10 + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "print(result.final_result.choices[0].message.content)", - "id": "8d537f80a6cfe6e4" + "id": "8d537f80a6cfe6e4", + "metadata": {}, + "outputs": [], + "source": [ + "print(result.final_result.choices[0].message.content)" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "719b5e0b7627671b", + "metadata": {}, "source": [ "(content_filtering)=\n", "### Content Filtering\n", "\n", "The `Content Filtering` module can be configured to filter both the `input` to the LLM module (input filter) and the `output` generated by the LLM (output filter). The module uses predefined classification services to detect inappropriate or unwanted content. Azure Content Filter sensitivity is controlled by customizable `thresholds`, assuring the content aligns with the desired standards before processing or generating as output. Llama Guard 3 Filter, equipped with 14 categories, runs on a binary mechanism, accepting only true or false. Setting a category to true enables filtering for it. It's possible to execute both filters in a single request, optimizing efficiency." - ], - "id": "719b5e0b7627671b" + ] }, { + "cell_type": "code", + "execution_count": 11, + "id": "e5743fda48a03b", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:25:18.924262Z", "start_time": "2026-03-19T07:25:18.893833Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import (AzureContentSafetyInput, AzureContentSafetyOutput, AzureThreshold,\n", " LlamaGuard38bFilter, FilteringModuleConfig, InputFiltering, OutputFiltering,\n", @@ -567,14 +584,14 @@ "config = OrchestrationConfig(modules=module_config)\n", "\n", "client = OrchestrationService(config=config)" - ], - "id": "e5743fda48a03b", - "outputs": [], - "execution_count": 11 + ] }, { - "metadata": {}, "cell_type": "code", + "execution_count": null, + "id": "46f631d9f77711f0", + "metadata": {}, + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import OrchestrationError\n", "\n", @@ -583,35 +600,35 @@ " print(result.final_result.choices[0].message.content)\n", "except OrchestrationError as er:\n", " print(er.message)" - ], - "id": "46f631d9f77711f0", - "outputs": [], - "execution_count": null + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "b383c26e816ccfb6", + "metadata": {}, "source": [ "(orchestration_streaming)=\n", "## Streaming\n", "\n", "When you initiate an orchestration request, the full response is typically processed and delivered in one go. For longer responses, this can lead to delays in receiving the complete output. To mitigate this, you have the option to stream the results as they are being generated. This helps in rapidly processing or displaying initial portions of the results without waiting for the entire computation to finish.\n" - ], - "id": "b383c26e816ccfb6" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "2a1f96083b525bcc", + "metadata": {}, "source": [ "To activate streaming, use the `stream` method of the `OrchestrationService` with the `stream` option in `OrchestrationConfig`. This method returns an object that streams chunks of the response as they become available. You can then extract relevant information from the `delta` field.\n", "\n", "Here's how you can set up a simple configuration to stream orchestration results:" - ], - "id": "2a1f96083b525bcc" + ] }, { - "metadata": {}, "cell_type": "code", + "execution_count": null, + "id": "7520e9502ae2e0ec", + "metadata": {}, + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import GlobalStreamOptions\n", "\n", @@ -646,28 +663,30 @@ "for part in result:\n", " print(part.final_result.choices[0].delta.content)\n", " print(\"*\" * 20)\n" - ], - "id": "7520e9502ae2e0ec", - "outputs": [], - "execution_count": null + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "**Note:** As shown above, streaming responses contain a delta field instead of a message field.", - "id": "e010edf32f8f6b94" + "id": "e010edf32f8f6b94", + "metadata": {}, + "source": [ + "**Note:** As shown above, streaming responses contain a delta field instead of a message field." + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "You can customize the global stream behavior by setting options like `chunk_size` which controls the amount of data processed in each chunk:", - "id": "dd2376220db6da05" + "id": "dd2376220db6da05", + "metadata": {}, + "source": [ + "You can customize the global stream behavior by setting options like `chunk_size` which controls the amount of data processed in each chunk:" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "bc153c49faaf43e0", + "metadata": {}, + "outputs": [], "source": [ "config = OrchestrationConfig(modules=module_config,\n", " stream=GlobalStreamOptions(enabled=True, chunk_size=25))\n", @@ -680,20 +699,22 @@ "for part in result:\n", " print(part.final_result.choices[0].delta.content)\n", " print(\"*\" * 20)" - ], - "id": "bc153c49faaf43e0" + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "Modules that influence or process streaming results, such as `OutputFiltering`, might need specific stream options. The `overlap` option allows you to include extra context during the filtering process:", - "id": "f9cc085d9148fa6a" + "id": "f9cc085d9148fa6a", + "metadata": {}, + "source": [ + "Modules that influence or process streaming results, such as `OutputFiltering`, might need specific stream options. The `overlap` option allows you to include extra context during the filtering process:" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "f9173122149a3e12", + "metadata": {}, + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import FilteringStreamOptions\n", "content_filter_config = FilteringModuleConfig(\n", @@ -725,12 +746,12 @@ "\n", "for chunk in response:\n", " print(chunk.final_result.choices[0].delta.content, end='')\n" - ], - "id": "f9173122149a3e12" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "f5e6ca171dc54a97", + "metadata": {}, "source": [ "(tool_calling)=\n", "## Tool Calling (Function Calling)\n", @@ -748,17 +769,19 @@ "#### Using the Python Decorator\n", "\n", "The simplest way to define a tool is to decorate a Python function with `@function_tool()`. The function’s signature and docstring are used to describe the tool to the LLM." - ], - "id": "f5e6ca171dc54a97" + ] }, { + "cell_type": "code", + "execution_count": 14, + "id": "70b23e987fda41f1", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:26:07.354117Z", "start_time": "2026-03-19T07:26:07.348749Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import function_tool\n", "\n", @@ -773,29 +796,29 @@ " return a + b\n", "\n", "tools = [multiply, add]" - ], - "id": "70b23e987fda41f1", - "outputs": [], - "execution_count": 14 + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "e558dbca183e48a7", + "metadata": {}, "source": [ "#### Using the `FunctionTool` Class\n", "\n", "For more control, you can use the `FunctionTool` class directly. This is useful if you want to customize the schema, enable strict argument checking, or wrap an existing function." - ], - "id": "e558dbca183e48a7" + ] }, { + "cell_type": "code", + "execution_count": 15, + "id": "14f259a50d7a856c", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:26:14.152448Z", "start_time": "2026-03-19T07:26:14.150221Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import FunctionTool, FunctionObject\n", "\n", @@ -825,43 +848,43 @@ "weather_tool = FunctionTool(function=weather_tool_func)\n", "\n", "tools = [weather_tool]" - ], - "id": "14f259a50d7a856c", - "outputs": [], - "execution_count": 15 + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "You can also create a `FunctionTool` from a function using the `from_function` static method:", - "id": "b86d28307b869444" + "id": "b86d28307b869444", + "metadata": {}, + "source": [ + "You can also create a `FunctionTool` from a function using the `from_function` static method:" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "757ae1af213fedd8", + "metadata": {}, + "outputs": [], "source": [ "weather_tool = FunctionTool.from_function(get_weather, strict=True)\n", "tools = [weather_tool]" - ], - "id": "757ae1af213fedd8" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "85dfc74ee8420281", + "metadata": {}, "source": [ "#### Using a JSON Schema Dictionary\n", "\n", "You can define a tool directly as a JSON schema dictionary. This is useful if you want to specify the tool interface without implementing the function in Python, or if you want to integrate with external systems." - ], - "id": "85dfc74ee8420281" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "a2bd6ac0ce9d7e5c", + "metadata": {}, + "outputs": [], "source": [ "tools = [{\n", " \"type\": \"function\",\n", @@ -884,23 +907,27 @@ " \"strict\": True\n", " }\n", "}]" - ], - "id": "a2bd6ac0ce9d7e5c" + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "You can then attach any of these tool definitions to your template:", - "id": "5d85e6155ac2959b" + "id": "5d85e6155ac2959b", + "metadata": {}, + "source": [ + "You can then attach any of these tool definitions to your template:" + ] }, { + "cell_type": "code", + "execution_count": 16, + "id": "23d3a1974c34c1a", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:26:32.640345Z", "start_time": "2026-03-19T07:26:32.636307Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import Template, SystemMessage, UserMessage\n", "\n", @@ -911,24 +938,24 @@ " ],\n", " tools=tools,\n", ")" - ], - "id": "23d3a1974c34c1a", - "outputs": [], - "execution_count": 16 + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "59d9ddcf139f2e34", + "metadata": {}, "source": [ "### Synchronous Tool Call Workflow\n", "\n", "When the LLM decides to call a tool, the orchestration response will include a `tool_calls` field. You are responsible for executing the tool(s), adding the results to the conversation history, and running the orchestration again to get the final answer." - ], - "id": "59d9ddcf139f2e34" + ] }, { - "metadata": {}, "cell_type": "code", + "execution_count": null, + "id": "b8f342d04ef20f2b", + "metadata": {}, + "outputs": [], "source": [ "from typing import List\n", "from gen_ai_hub.orchestration_v2 import ChatMessage, SystemMessage, UserMessage, ToolChatMessage\n", @@ -971,26 +998,24 @@ " history=history,\n", ")\n", "print(response2.final_result.choices[0].message.content)" - ], - "id": "b8f342d04ef20f2b", - "outputs": [], - "execution_count": null + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "721d327f7e5615ed", + "metadata": {}, "source": [ "### Streaming Tool Calls\n", "\n", "When using streaming, tool calls may be split across multiple chunks. The `delta.tool_calls` field in each chunk contains partial or complete tool call information. You may need to buffer and concatenate arguments if they arrive in pieces." - ], - "id": "721d327f7e5615ed" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "ea2a364927c0dd41", + "metadata": {}, + "outputs": [], "source": [ "# Assume 'config' and 'service' are defined as above\n", "config = OrchestrationConfig(modules=module_config, stream=GlobalStreamOptions(enabled=True))\n", @@ -1009,12 +1034,12 @@ " final_tool_calls[index].function.arguments += tool_call.function.arguments\n", "\n", "# Now final_tool_calls contains all tool calls with complete arguments" - ], - "id": "ea2a364927c0dd41" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "9fed1b18ba749dff", + "metadata": {}, "source": [ "**⚠️ Note on Agentic Loop Support:**\n", "\n", @@ -1027,12 +1052,133 @@ "> - Re-invoking the orchestration service as needed\n", ">\n", "> This approach gives you maximum flexibility, but you must implement the orchestration loop logic yourself." - ], - "id": "9fed1b18ba749dff" + ] }, { + "cell_type": "markdown", + "id": "305c2f30", "metadata": {}, + "source": [ + "(prompt_caching)=\n", + "## Prompt Caching\n", + "\n", + "Cache control improves performance by caching requests at the model provider level.\n", + "Add a `cache_control` breakpoint to a content block to mark a cache breakpoint in the prompt.\n", + "Availability of different time-to-live (TTL) options may depend on the model.\n", + "\n", + "This feature covers explicit cache control for AWS Bedrock-hosted models (Anthropic Claude and Amazon Nova).\n", + "Other model providers, such as OpenAI and Gemini, use implicit context caching that is enabled by default\n", + "and requires no additional configuration.\n", + "\n", + "**Note:** Prompt caching requires a minimum number of tokens in the cached content.\n", + "If the prompt is too short, the cache breakpoint may not take effect.\n", + "\n", + "### Attaching `cache_control` to Content Blocks\n", + "\n", + "Set `cache_control` on individual content parts inside a message.\n", + "The example below caches a long system prompt and marks the last user turn as a second\n", + "cache breakpoint, following the few-shot pattern recommended for classification tasks.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "802dc96e", + "metadata": {}, + "outputs": [], + "source": [ + "from gen_ai_hub.orchestration_v2 import (\n", + " CacheControl, TextPart,\n", + " SystemMessage, UserMessage, AssistantMessage, Template,\n", + " LLMModelDetails, PromptTemplatingModuleConfig, ModuleConfig,\n", + " OrchestrationConfig, OrchestrationService,\n", + ")\n", + "\n", + "# A long system prompt whose tokens will be cached after the first request.\n", + "system_text = \"You are an expert news article classifier. Classify each article into exactly one category.\"\n", + "system_text += \" Classify each article into exactly one category.\" * 512 # repeat to exceed the minimum token threshold\n", + "\n", + "template = Template(\n", + " template=[\n", + " SystemMessage(\n", + " content=[\n", + " TextPart(\n", + " text=system_text,\n", + " cache_control=CacheControl(type=\"ephemeral\"),\n", + " )\n", + " ]\n", + " ),\n", + " # Few-shot examples (plain strings, not cached)\n", + " UserMessage(content=\"input: Comcast launches prepaid plans\"),\n", + " AssistantMessage(content=\"Business\"),\n", + " # Dynamic user turn: second cache breakpoint\n", + " UserMessage(\n", + " content=[\n", + " TextPart(\n", + " text=\"input: {{?article}}\",\n", + " cache_control=CacheControl(type=\"ephemeral\"),\n", + " )\n", + " ]\n", + " ),\n", + " ]\n", + ")\n", + "\n", + "llm = LLMModelDetails(name=\"anthropic--claude-4.5-sonnet\")\n", + "\n", + "prompt_template = PromptTemplatingModuleConfig(prompt=template, model=llm)\n", + "module_config = ModuleConfig(prompt_templating=prompt_template)\n", + "config = OrchestrationConfig(modules=module_config)\n", + "\n", + "orchestration_service = OrchestrationService(config=config)\n", + "\n", + "result = orchestration_service.run(\n", + " placeholder_values={\n", + " \"article\": \"Scaling up neural models has yielded significant advancements in language generation\"\n", + " }\n", + ")\n", + "print(result.final_result.choices[0].message.content)" + ] + }, + { "cell_type": "markdown", + "id": "3ed1fb84", + "metadata": {}, + "source": [ + "### Inspecting Cache Token Usage\n", + "\n", + "For supported models the `prompt_tokens_details` field on the usage object reports how many\n", + "tokens were written to the cache (`cache_creation_tokens`) and how many were read from it\n", + "(`cached_tokens`). On a warm cache at least one of the two values will be greater than zero." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "58e218fb", + "metadata": {}, + "outputs": [], + "source": [ + "usage = result.final_result.usage\n", + "details = usage.prompt_tokens_details\n", + "print(f\"cache_creation_tokens : {details.cache_creation_tokens}\")\n", + "print(f\"cached_tokens : {details.cached_tokens}\")" + ] + }, + { + "cell_type": "markdown", + "id": "1b4585d4", + "metadata": {}, + "source": [ + "### Attaching `cache_control` to a Tool Definition\n", + "\n", + "Some models also support caching tool definitions.\n", + "Pass `cache_control` directly to the `FunctionTool` constructor." + ] + }, + { + "cell_type": "markdown", + "id": "c34749a2a7157b19", + "metadata": {}, "source": [ "(input_images)=\n", "## Using Images as Input\n", @@ -1053,17 +1199,19 @@ "* **Data URL:** Provide the image data directly embedded in the URL string.\n", "\n", "**Note:** For web URLs, ensure the image is publicly accessible, as the service will need to fetch it." - ], - "id": "c34749a2a7157b19" + ] }, { + "cell_type": "code", + "execution_count": 18, + "id": "f6c9ee8bbe87618e", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:27:20.299676Z", "start_time": "2026-03-19T07:27:20.296308Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import ImageItem\n", "\n", @@ -1077,27 +1225,25 @@ "image_from_data_url = ImageItem(\n", " url=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAIAAAACUFjqAAAAE0lEQVR4nGP8z4APMOGVZRip0gBBLAETee26JgAAAABJRU5ErkJggg==\"\n", ")" - ], - "id": "f6c9ee8bbe87618e", - "outputs": [], - "execution_count": 18 + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "a1409812f52d7ef2", + "metadata": {}, "source": [ "#### b) From a Local File\n", "\n", "If your image resides on your local filesystem, you can load it directly using the `ImageItem.from_file()` class method.\n", "The `from_file` method handles opening, reading, and base64 encoding the image data for you, packaging it into an `ImageItem`." - ], - "id": "a1409812f52d7ef2" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "2abef5e8ef2ccf60", + "metadata": {}, + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import ImageItem\n", "\n", @@ -1109,22 +1255,24 @@ " print(\"Error: The specified image file was not found.\")\n", "except Exception as e:\n", " print(f\"An error occurred while loading the image: {e}\")" - ], - "id": "2abef5e8ef2ccf60" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "78f150ec68206f25", + "metadata": {}, "source": [ "### 2. Adding Images to a Prompt\n", "\n", "Once you have your `ImageItem` object(s), you can combine them with text to create a multimodal prompt. This is done by passing a list containing `ImageItem` instances and text strings to the `content` parameter of a `UserMessage`." - ], - "id": "78f150ec68206f25" + ] }, { - "metadata": {}, "cell_type": "code", + "execution_count": null, + "id": "595c6615905e4519", + "metadata": {}, + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import UserMessage\n", "\n", @@ -1145,29 +1293,29 @@ "service = OrchestrationService(config=config)\n", "response = service.run()\n", "print(response.final_result.choices[0].message.content)" - ], - "id": "595c6615905e4519", - "outputs": [], - "execution_count": null + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "e00e2ddabfa857d0", + "metadata": {}, "source": [ "(translation)=\n", "## Translation\n", "Translation module can be used to translate text from one language to another. You can use this module to translate input text before it is processed by the LLM module, or to translate the output generated by the LLM module. The translation module uses the SAP Document Translation service to perform the translation." - ], - "id": "e00e2ddabfa857d0" + ] }, { + "cell_type": "code", + "execution_count": 20, + "id": "249efe8e5e9cc60f", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:28:43.144994Z", "start_time": "2026-03-19T07:28:43.113534Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import (SystemMessage, UserMessage, Template, PromptTemplatingModuleConfig,\n", " LLMModelDetails, ModuleConfig, OrchestrationConfig, OrchestrationService,\n", @@ -1207,64 +1355,70 @@ "config = OrchestrationConfig(modules=module_config)\n", "\n", "orchestration_service = OrchestrationService()" - ], - "id": "249efe8e5e9cc60f", - "outputs": [], - "execution_count": 20 + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "fb96a0b39c73e145", + "metadata": {}, + "outputs": [], "source": [ "result = orchestration_service.run(\n", " config=config,\n", " placeholder_values={\"text\": \"What is the capital of Germany?\"}\n", ")" - ], - "id": "fb96a0b39c73e145" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "print(result.final_result.choices[0].message.content)", - "id": "9faa54034fc077a" + "id": "9faa54034fc077a", + "metadata": {}, + "outputs": [], + "source": [ + "print(result.final_result.choices[0].message.content)" + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "## Advanced Examples", - "id": "5aec4fd632c3a0fd" + "id": "5aec4fd632c3a0fd", + "metadata": {}, + "source": [ + "## Advanced Examples" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "service = OrchestrationService(api_url=YOUR_API_URL)", - "id": "d2a6a1c7f7e3aad3" + "id": "d2a6a1c7f7e3aad3", + "metadata": {}, + "outputs": [], + "source": [ + "service = OrchestrationService(api_url=YOUR_API_URL)" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "d05456d9685578c6", + "metadata": {}, "source": [ "### Translation Service\n", "\n", "This example extends the initial walkthrough of a basic orchestration pipeline by abstracting the translation task into its own reusable `TranslationService` class. Once the configuration is established, it can be easily adapted and reused for different translation scenarios." - ], - "id": "d05456d9685578c6" + ] }, { + "cell_type": "code", + "execution_count": 21, + "id": "f0c675fc9f10b823", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:29:31.451027Z", "start_time": "2026-03-19T07:29:31.438376Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import (OrchestrationConfig, ModuleConfig, LLMModelDetails, SystemMessage, UserMessage,\n", " Template, PromptTemplatingModuleConfig, OrchestrationService)\n", @@ -1299,55 +1453,55 @@ " )\n", "\n", " return response.final_result.choices[0].message.content\n" - ], - "id": "f0c675fc9f10b823", - "outputs": [], - "execution_count": 21 + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "translator = TranslationService(orchestration_service=service)", - "id": "4b1507b5111848dc" + "id": "4b1507b5111848dc", + "metadata": {}, + "outputs": [], + "source": [ + "translator = TranslationService(orchestration_service=service)" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "15bf0d86f3404076", + "metadata": {}, + "outputs": [], "source": [ "result = translator.translate(text=\"Hello, world!\", to_lang=\"French\")\n", "print(result)" - ], - "id": "15bf0d86f3404076" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "9134210a296d1a60", + "metadata": {}, + "outputs": [], "source": [ "result = translator.translate(text=\"Hello, world!\", to_lang=\"Spanish\")\n", "print(result)" - ], - "id": "9134210a296d1a60" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "53f4af1db029400", + "metadata": {}, + "outputs": [], "source": [ "result = translator.translate(text=\"Hello, world!\", to_lang=\"German\")\n", "print(result)" - ], - "id": "53f4af1db029400" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "1a31cbd3f644327d", + "metadata": {}, "source": [ "### Chatbot with Memory\n", "\n", @@ -1356,17 +1510,19 @@ "When making requests to the orchestration service, you can specify a list of messages as `history` that will be prepended to the templated content and processed by the templating module. These messages are plain, non-templated messages, as they typically represent past conversation outputs — such as in this chatbot scenario.\n", "\n", "It’s important to note that managing conversation history / state is handled locally in the `ChatBot` class, not by the orchestration service itself." - ], - "id": "1a31cbd3f644327d" + ] }, { + "cell_type": "code", + "execution_count": 22, + "id": "797481251e803a8", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:30:01.880952Z", "start_time": "2026-03-19T07:30:01.874120Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from typing import List\n", "\n", @@ -1410,62 +1566,72 @@ "\n", " def reset(self):\n", " self.history = []" - ], - "id": "797481251e803a8", - "outputs": [], - "execution_count": 22 + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "bot = ChatBot(orchestration_service=OrchestrationService())", - "id": "2203c5a8987476d0" + "id": "2203c5a8987476d0", + "metadata": {}, + "outputs": [], + "source": [ + "bot = ChatBot(orchestration_service=OrchestrationService())" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "print(bot.chat(\"Hello, how are you?\"))", - "id": "e6a9a551086be501" + "id": "e6a9a551086be501", + "metadata": {}, + "outputs": [], + "source": [ + "print(bot.chat(\"Hello, how are you?\"))" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "print(bot.chat(\"What's the weather like today?\"))", - "id": "55b89a379f235062" + "id": "55b89a379f235062", + "metadata": {}, + "outputs": [], + "source": [ + "print(bot.chat(\"What's the weather like today?\"))" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "print(bot.chat(\"Can you remember what I first asked you?\"))", - "id": "1e0cbe9d7029aa5b" + "id": "1e0cbe9d7029aa5b", + "metadata": {}, + "outputs": [], + "source": [ + "print(bot.chat(\"Can you remember what I first asked you?\"))" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "bot.reset()", - "id": "eebcf6592d2f4342" + "id": "eebcf6592d2f4342", + "metadata": {}, + "outputs": [], + "source": [ + "bot.reset()" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "print(bot.chat(\"Can you remember what I first asked you?\"))", - "id": "adb08617bb2073b3" + "id": "adb08617bb2073b3", + "metadata": {}, + "outputs": [], + "source": [ + "print(bot.chat(\"Can you remember what I first asked you?\"))" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "37bcd78e032fbae", + "metadata": {}, "source": [ "### Sentiment Analysis with Few Shot Learning \n", "\n", @@ -1477,17 +1643,19 @@ "\n", "\n", "The FewShotLearner class manages the dynamic creation of the template and ensures the correct message roles are used for each user input." - ], - "id": "37bcd78e032fbae" + ] }, { + "cell_type": "code", + "execution_count": 23, + "id": "87097426ad52e2c2", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:31:13.521272Z", "start_time": "2026-03-19T07:31:13.515516Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from typing import List, Tuple\n", "\n", @@ -1537,30 +1705,28 @@ " )\n", "\n", " return response.final_result.choices[0].message.content" - ], - "id": "87097426ad52e2c2", - "outputs": [], - "execution_count": 23 + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "13cc0919eb1232e0", + "metadata": {}, + "outputs": [], "source": [ "sentiment_examples = [\n", " (UserMessage(content=\"I love this product!\"), AssistantMessage(content=\"Positive\")),\n", " (UserMessage(content=\"This is terrible service.\"), AssistantMessage(content=\"Negative\")),\n", " (UserMessage(content=\"The weather is okay today.\"), AssistantMessage(content=\"Neutral\")),\n", "]" - ], - "id": "13cc0919eb1232e0" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "d04872dcb560e15c", + "metadata": {}, + "outputs": [], "source": [ "sentiment_analyzer = FewShotLearner(\n", " orchestration_service=OrchestrationService(),\n", @@ -1569,44 +1735,46 @@ " ),\n", " examples=sentiment_examples,\n", ")" - ], - "id": "d04872dcb560e15c" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "print(sentiment_analyzer.predict(\"The movie was a complete waste of time!\"))", - "id": "75b35fb528b2f318" + "id": "75b35fb528b2f318", + "metadata": {}, + "outputs": [], + "source": [ + "print(sentiment_analyzer.predict(\"The movie was a complete waste of time!\"))" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "33eeea427b8937f0", + "metadata": {}, + "outputs": [], "source": [ "print(\n", " sentiment_analyzer.predict(\"The traffic was fortunately unusually light today.\")\n", ")" - ], - "id": "33eeea427b8937f0" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "65b86a4daa668515", + "metadata": {}, + "outputs": [], "source": [ "print(\n", " sentiment_analyzer.predict(\"I'm not sure how I feel about the recent events.\")\n", ")" - ], - "id": "65b86a4daa668515" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "51dfba0273406622", + "metadata": {}, "source": [ "(orchestration_async)=\n", "## Async Support\n", @@ -1615,17 +1783,19 @@ "Use:\n", "- `arun` from the async version of `run`\n", "- `astream` from the async version of `stream`" - ], - "id": "51dfba0273406622" + ] }, { + "cell_type": "code", + "execution_count": 24, + "id": "c8b1be600f3eff58", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:31:49.927838Z", "start_time": "2026-03-19T07:31:49.898436Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import (SystemMessage, UserMessage, Template, PromptTemplatingModuleConfig,\n", " LLMModelDetails, OrchestrationConfig, ModuleConfig)\n", @@ -1651,28 +1821,28 @@ "# Instantiate the orchestration service.\n", "from gen_ai_hub.orchestration_v2 import OrchestrationService\n", "orchestration_service = OrchestrationService(config=config)\n" - ], - "id": "c8b1be600f3eff58", - "outputs": [], - "execution_count": 24 + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "669448831d08f18", + "metadata": {}, + "outputs": [], "source": [ "async def test_async():\n", " async_result = await orchestration_service.arun()\n", " display(Markdown(async_result.final_result.choices[0].message.content))\n", "\n", "await test_async()" - ], - "id": "669448831d08f18" + ] }, { - "metadata": {}, "cell_type": "code", + "execution_count": null, + "id": "101d32c4d15c28fc", + "metadata": {}, + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import GlobalStreamOptions\n", "\n", @@ -1685,14 +1855,12 @@ " display(Markdown(streamed_content))\n", "\n", "await test_streaming_async()" - ], - "id": "101d32c4d15c28fc", - "outputs": [], - "execution_count": null + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "8f64408b86b9aac7", + "metadata": {}, "source": [ "(orchestration_embeddings)=\n", "## Embeddings\n", @@ -1704,22 +1872,24 @@ "- **RAG (Retrieval-Augmented Generation)**: Retrieve relevant context for LLM prompts\n", "- **Document Clustering**: Group similar documents together\n", "- **Similarity Comparison**: Measure how semantically similar two texts are" - ], - "id": "8f64408b86b9aac7" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "eed7aac13f9af34b", + "metadata": {}, "source": [ "### Basic Usage\n", "\n", "Generate an embedding for a single text string with minimal configuration." - ], - "id": "eed7aac13f9af34b" + ] }, { - "metadata": {}, "cell_type": "code", + "execution_count": null, + "id": "14d32266c44b3d41", + "metadata": {}, + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import (OrchestrationService, EmbeddingsOrchestrationConfig, EmbeddingsModuleConfigs,\n", " EmbeddingsModelConfig, EmbeddingsModelDetails, EmbeddingsInput)\n", @@ -1743,14 +1913,12 @@ "embedding = response.final_result.data[0].embedding\n", "print(f\"Embedding dimensions: {len(embedding)}\")\n", "print(f\"First 5 values: {embedding[:5]}\")" - ], - "id": "14d32266c44b3d41", - "outputs": [], - "execution_count": null + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "6c2e0fec7a1bfb4d", + "metadata": {}, "source": [ "### Customizing Embedding Parameters\n", "\n", @@ -1761,12 +1929,14 @@ "| `dimensions` | Number of dimensions in the output | e.g. 256, 512, 1536, 3072 |\n", "| `encoding_format` | Output format | `FLOAT`, `BASE64`, `BINARY` |\n", "| `normalize` | Normalize the vector | `True`, `False` |" - ], - "id": "6c2e0fec7a1bfb4d" + ] }, { - "metadata": {}, "cell_type": "code", + "execution_count": null, + "id": "18ec9508f808ca86", + "metadata": {}, + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import EmbeddingsModelParams, EmbeddingsEncodingFormat\n", "\n", @@ -1791,26 +1961,24 @@ ")\n", "\n", "print(f\"Embedding dimensions: {len(response.final_result.data[0].embedding)}\")" - ], - "id": "18ec9508f808ca86", - "outputs": [], - "execution_count": null + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "304bfc990c316b5f", + "metadata": {}, "source": [ "### Batch Embeddings\n", "\n", "Generate embeddings for multiple texts in a single request for better efficiency." - ], - "id": "304bfc990c316b5f" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "a3fbcc6bb228c9b6", + "metadata": {}, + "outputs": [], "source": [ "documents = [\n", " \"Artificial intelligence is transforming industries worldwide.\",\n", @@ -1827,12 +1995,12 @@ "print(f\"Generated {len(response.final_result.data)} embeddings\")\n", "for result in response.final_result.data:\n", " print(f\" Index {result.index}: {len(result.embedding)} dimensions\")" - ], - "id": "a3fbcc6bb228c9b6" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "77ce13be4c12e88c", + "metadata": {}, "source": [ "### Input Type Hints (Asymmetric Search)\n", "\n", @@ -1843,17 +2011,19 @@ "| `TEXT` | General purpose (default) |\n", "| `DOCUMENT` | Content to be indexed and searched |\n", "| `QUERY` | Search queries to find relevant documents |" - ], - "id": "77ce13be4c12e88c" + ] }, { + "cell_type": "code", + "execution_count": 29, + "id": "16713afef9aaf491", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:36:14.961516Z", "start_time": "2026-03-19T07:36:14.113464Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import EmbeddingsInputType\n", "\n", @@ -1874,24 +2044,24 @@ " type=EmbeddingsInputType.QUERY\n", " )\n", ")" - ], - "id": "16713afef9aaf491", - "outputs": [], - "execution_count": 29 + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "be66c74b54173af4", + "metadata": {}, "source": [ "### Embeddings with Data Masking\n", "\n", "When embedding sensitive data, use the data masking module to anonymize PII before generating embeddings. This ensures sensitive information is not exposed to the embedding model." - ], - "id": "be66c74b54173af4" + ] }, { - "metadata": {}, "cell_type": "code", + "execution_count": null, + "id": "466214c78e1cac56", + "metadata": {}, + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import (MaskingModuleConfig, MaskingMethod, MaskingProviderConfig, DPIStandardEntity,\n", " ProfileEntity)\n", @@ -1926,29 +2096,29 @@ "print(f\"Embedding generated with PII masked\")\n", "print(f\"Intermediate results: {response.intermediate_results}\")\n", "print(f\"Dimensions: {len(response.final_result.data[0].embedding)}\")" - ], - "id": "466214c78e1cac56", - "outputs": [], - "execution_count": null + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "8b862c98f3f31303", + "metadata": {}, "source": [ "#### Masking with Custom Entities and Allowlist\n", "\n", "Use regular expressions to mask custom patterns and allowlists to exclude specific terms from masking." - ], - "id": "8b862c98f3f31303" + ] }, { + "cell_type": "code", + "execution_count": 31, + "id": "8f559491b4487002", "metadata": { "ExecuteTime": { "end_time": "2026-03-19T07:36:55.310595Z", "start_time": "2026-03-19T07:36:53.720489Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from gen_ai_hub.orchestration_v2 import DPICustomEntity, DPIMethodConstant\n", "\n", @@ -1987,26 +2157,24 @@ " text=\"Employee John Doe (ID: 89-SAP-550) works at SAP with Microsoft partners.\"\n", " )\n", ")" - ], - "id": "8f559491b4487002", - "outputs": [], - "execution_count": 31 + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "dea5a8ed3ab7d256", + "metadata": {}, "source": [ "### Async Embeddings\n", "\n", "For non-blocking operations, use the async `aembed` method." - ], - "id": "dea5a8ed3ab7d256" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "caa35ae2d459f1c", + "metadata": {}, + "outputs": [], "source": [ "async def embed_async():\n", " async_service = OrchestrationService()\n", @@ -2020,21 +2188,20 @@ " await async_service.aclose_http_connection()\n", "\n", "await embed_async()" - ], - "id": "caa35ae2d459f1c" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "", - "id": "85e7b848963d74b2" + "id": "85e7b848963d74b2", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "env", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -2048,7 +2215,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.8" + "version": "3.11.8" } }, "nbformat": 4, From cb9d0f884a6cb410cb8a7b88c8a9d259f6491a0e Mon Sep 17 00:00:00 2001 From: Yamac Ay Date: Thu, 17 Sep 2026 15:24:51 +0200 Subject: [PATCH 09/11] fix: downgrade to anthropic 4.5 sonnet --- .../orchestration_v2/test_cache_control.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/gen/integration_tests/orchestration_v2/test_cache_control.py b/packages/gen/integration_tests/orchestration_v2/test_cache_control.py index c6ff8eeb..cba9d46a 100644 --- a/packages/gen/integration_tests/orchestration_v2/test_cache_control.py +++ b/packages/gen/integration_tests/orchestration_v2/test_cache_control.py @@ -2,7 +2,7 @@ Live integration tests for prompt caching (cache_control) via Orchestration V2. Caching is supported for Anthropic Claude and Amazon Nova models. -The tests target anthropic--claude-4.6-sonnet (1024-token minimum, 5m and 1h TTLs). +The tests target anthropic--claude-4.5-sonnet (1024-token minimum, 5m and 1h TTLs). Wire path: ai-sdk-python -> SAP AI Core /v2/completion -> SAP LiteLLM fork -> Anthropic API @@ -25,7 +25,7 @@ from integration_tests.orchestration_v2.test_base import OrchestrationServiceTestBase from integration_tests.test_helpers import retry_on_429_or_503 -# Must exceed the 1024-token minimum for claude-4.6-sonnet cache points. +# Must exceed the 1024-token minimum for claude-4.5-sonnet cache points. _LONG_SYSTEM_PROMPT = ( "You are a helpful assistant with deep knowledge of European history. " "Below is a detailed reference text that you must use to answer questions accurately.\n\n" @@ -58,7 +58,7 @@ ) _LLM = LLMModelDetails( - name="anthropic--claude-4.6-sonnet", + name="anthropic--claude-4.5-sonnet", params={"max_tokens": 64, "temperature": 0.0}, ) From 6bd800c7105716d5beeb82eda6cfd0022ebee6d0 Mon Sep 17 00:00:00 2001 From: Yamac Ay Date: Thu, 17 Sep 2026 16:09:02 +0200 Subject: [PATCH 10/11] fix: updated example with function tool --- .../examples/orchestration-service2.ipynb | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/packages/gen/docs/gen_ai_hub/examples/orchestration-service2.ipynb b/packages/gen/docs/gen_ai_hub/examples/orchestration-service2.ipynb index a34a2ae2..579a3a35 100644 --- a/packages/gen/docs/gen_ai_hub/examples/orchestration-service2.ipynb +++ b/packages/gen/docs/gen_ai_hub/examples/orchestration-service2.ipynb @@ -919,7 +919,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "23d3a1974c34c1a", "metadata": { "ExecuteTime": { @@ -1175,6 +1175,46 @@ "Pass `cache_control` directly to the `FunctionTool` constructor." ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d8dc687", + "metadata": {}, + "outputs": [], + "source": [ + "from gen_ai_hub.orchestration_v2 import FunctionTool, FunctionObject\n", + "\n", + "def get_weather(location: str) -> str:\n", + " \"\"\"Get current temperature for a given location.\"\"\"\n", + " # Replace with your actual implementation\n", + " return \"22°C\"\n", + "\n", + "weather_tool_func = FunctionObject(\n", + " name=\"get_weather\",\n", + " description=\"Get current temperature for a given location.\",\n", + " parameters={\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"location\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"City and country e.g. Bogotá, Colombia\"\n", + " }\n", + " },\n", + " \"required\": [\"location\"],\n", + " \"additionalProperties\": False\n", + " },\n", + " strict=True,\n", + " function=get_weather\n", + ")\n", + "\n", + "weather_tool = FunctionTool(\n", + " function=weather_tool_func, \n", + " cache_control=CacheControl(type=\"ephemeral\"),\n", + ")\n", + "\n", + "tools = [weather_tool]" + ] + }, { "cell_type": "markdown", "id": "c34749a2a7157b19", From c7c7cffcab1c118ddc7394ee06a652b6102d10cb Mon Sep 17 00:00:00 2001 From: Yamac Ay Date: Thu, 17 Sep 2026 18:39:21 +0200 Subject: [PATCH 11/11] added function tool example --- .../examples/orchestration-service2.ipynb | 155 +++++++++++++++--- 1 file changed, 135 insertions(+), 20 deletions(-) diff --git a/packages/gen/docs/gen_ai_hub/examples/orchestration-service2.ipynb b/packages/gen/docs/gen_ai_hub/examples/orchestration-service2.ipynb index 579a3a35..52a6e219 100644 --- a/packages/gen/docs/gen_ai_hub/examples/orchestration-service2.ipynb +++ b/packages/gen/docs/gen_ai_hub/examples/orchestration-service2.ipynb @@ -963,7 +963,7 @@ "\n", "# Assume 'template' and 'weather_tool' are defined as above\n", "llm = LLMModelDetails(name=\"gpt-4o-mini\", params={\"max_completion_tokens\": 200, \"temperature\": 0.0})\n", - "rompt_template = PromptTemplatingModuleConfig(prompt=template,\n", + "prompt_template = PromptTemplatingModuleConfig(prompt=template,\n", " model=llm)\n", "module_config = ModuleConfig(prompt_templating=prompt_template)\n", "\n", @@ -1182,37 +1182,152 @@ "metadata": {}, "outputs": [], "source": [ - "from gen_ai_hub.orchestration_v2 import FunctionTool, FunctionObject\n", + "from gen_ai_hub.orchestration_v2 import FunctionTool, FunctionObject, CacheControl\n", "\n", "def get_weather(location: str) -> str:\n", " \"\"\"Get current temperature for a given location.\"\"\"\n", " # Replace with your actual implementation\n", " return \"22°C\"\n", "\n", - "weather_tool_func = FunctionObject(\n", - " name=\"get_weather\",\n", - " description=\"Get current temperature for a given location.\",\n", - " parameters={\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"location\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"City and country e.g. Bogotá, Colombia\"\n", - " }\n", + "def get_climate_report(location: str, start_date: str, end_date: str, **kwargs) -> str:\n", + " \"\"\"Get a verbose climate report for a given location and time interval\"\"\"\n", + " # Replace with your actual implementation\n", + " return \"{}\"\n", + "\n", + "weather_tool = FunctionTool(\n", + " function=FunctionObject(\n", + " name=\"get_weather\",\n", + " description=\"Get the current temperature for a given location.\",\n", + " parameters={\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"location\": {\"type\": \"string\", \"description\": \"City and country e.g. Bogotá, Colombia\"},\n", + " },\n", + " \"required\": [\"location\"],\n", + " \"additionalProperties\": False,\n", " },\n", - " \"required\": [\"location\"],\n", - " \"additionalProperties\": False\n", - " },\n", - " strict=True,\n", - " function=get_weather\n", + " strict=True,\n", + " function=get_weather,\n", + " ),\n", ")\n", "\n", - "weather_tool = FunctionTool(\n", - " function=weather_tool_func, \n", + "# Verbose tool definition, pushing total schema past Anthropic's 1024-token cache minimum.\n", + "# cache_control goes on the last tool in the array, caching the entire tools prefix.\n", + "climate_tool = FunctionTool(\n", + " function=FunctionObject(\n", + " name=\"get_climate_report\",\n", + " description=(\n", + " \"Retrieves a comprehensive climate and environmental report for a specified location \"\n", + " \"and date range. The report covers historical weather patterns, seasonal temperature \"\n", + " \"trends, air quality indices (AQI), UV exposure levels, precipitation forecasts, \"\n", + " \"relative humidity statistics, wind speed and direction analysis, atmospheric pressure \"\n", + " \"readings, pollen counts by category (tree, grass, weed), and general climate \"\n", + " \"advisories issued by local meteorological authorities. Intended for environmental \"\n", + " \"researchers, travel planners, outdoor event organizers, agricultural analysts, and \"\n", + " \"health-conscious users requiring detailed atmospheric data beyond simple temperature.\"\n", + " ) * 6,\n", + " parameters={\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"location\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"Target location as city and country e.g. Berlin, Germany.\",\n", + " },\n", + " \"start_date\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"Start of the reporting period in ISO 8601 format (YYYY-MM-DD).\",\n", + " },\n", + " \"end_date\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"End of the reporting period in ISO 8601 format (YYYY-MM-DD).\",\n", + " },\n", + " \"metrics\": {\n", + " \"type\": \"array\",\n", + " \"items\": {\"type\": \"string\"},\n", + " \"description\": (\n", + " \"Subset of metrics to include. Accepted values: temperature, humidity, \"\n", + " \"precipitation, wind_speed, wind_direction, aqi, uv_index, pressure, \"\n", + " \"pollen_tree, pollen_grass, pollen_weed. Omit to retrieve all metrics.\"\n", + " ),\n", + " },\n", + " \"units\": {\n", + " \"type\": \"string\",\n", + " \"enum\": [\"metric\", \"imperial\"],\n", + " \"description\": \"Unit system for numerical values. Defaults to metric.\",\n", + " },\n", + " \"interval\": {\n", + " \"type\": \"string\",\n", + " \"enum\": [\"hourly\", \"daily\", \"weekly\"],\n", + " \"description\": \"Temporal resolution of the returned data series. Defaults to daily.\",\n", + " },\n", + " \"include_advisories\": {\n", + " \"type\": \"boolean\",\n", + " \"description\": (\n", + " \"When true, appends any active climate or health advisories issued for \"\n", + " \"the location within the requested date range.\"\n", + " ),\n", + " },\n", + " \"language\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"BCP 47 language tag for advisory text e.g. en, de, es. Defaults to en.\",\n", + " },\n", + " },\n", + " \"required\": [\"location\", \"start_date\", \"end_date\"],\n", + " \"additionalProperties\": False,\n", + " },\n", + " strict=False,\n", + " function=get_climate_report,\n", + " ),\n", " cache_control=CacheControl(type=\"ephemeral\"),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "b4551acb", + "metadata": {}, + "source": [ + "The example below demonstrates prompt caching in a two-turn tool-call conversation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f480551", + "metadata": {}, + "outputs": [], + "source": [ + "from gen_ai_hub.orchestration_v2 import Template, SystemMessage, UserMessage\n", + "\n", + "template = Template(\n", + " template=[\n", + " SystemMessage(content=\"You are a weather assistant.\"),\n", + " UserMessage(content=\"What is the temperature in {{?location}}?\"),\n", + " ],\n", + " tools=[weather_tool, climate_tool],\n", ")\n", "\n", - "tools = [weather_tool]" + "config = OrchestrationConfig(\n", + " modules=ModuleConfig(\n", + " prompt_templating=PromptTemplatingModuleConfig(\n", + " prompt=template,\n", + " model=LLMModelDetails(name=\"anthropic--claude-4.6-sonnet\", params={\"max_tokens\": 50}),\n", + " )\n", + " )\n", + ")\n", + "\n", + "service = OrchestrationService()\n", + "placeholder_values = {\"location\": \"Bogotá, Colombia\"}\n", + "\n", + "# Run 1: writes the cache (cache_creation_tokens > 0, cached_tokens = 0)\n", + "response1 = service.run(config=config, placeholder_values=placeholder_values)\n", + "u1 = response1.final_result.usage.prompt_tokens_details\n", + "print(f\"Run 1 — cache_creation_tokens: {u1.cache_creation_tokens}, cached_tokens: {u1.cached_tokens}\")\n", + "\n", + "# Run 2: hits the cache (cache_creation_tokens = 0, cached_tokens > 0)\n", + "response2 = service.run(config=config, placeholder_values=placeholder_values)\n", + "u2 = response2.final_result.usage.prompt_tokens_details\n", + "print(f\"Run 2 — cache_creation_tokens: {u2.cache_creation_tokens}, cached_tokens: {u2.cached_tokens}\")" ] }, {