How can I avoid logging images when using ChatBedrockConverse or other Bedrock related endpoints?

[Kunal (km-mesh)]

I need to prevent the logging of the images, but maintain the count of images for a prompt that contains both the images as well as text. I want the images(assuming 2 were passed in the prompt) to look like <IMG1> <IMG2> inside the trace.

I tried passing a custom LangSmith Client via RunnableConfig to the invoke call, but it didn't respect that. what can I do?

[Kunal (km-mesh)]

Some bits to look at:

def hide_images_from_langsmith(inputs: dict) -> dict:
    """
    Scans LangChain messages and replaces base64 image data with a placeholder.
    """
    if "messages" not in inputs:
        return inputs

    new_messages = []
    image_counter = 1

    for msg in inputs["messages"]:
        # LangChain messages can be BaseMessage objects or dicts
        content = getattr(msg, "content", None) or msg.get("content")

        if isinstance(content, list):
            new_content = []
            for item in content:
                # Bedrock/Converse format uses 'type': 'image'
                if isinstance(item, dict) and item.get("type") == "image":
                    new_content.append(
                        {"type": "text", "text": f"<IMAGE_DATA_HIDDEN_{image_counter}>"}
                    )
                    image_counter += 1
                else:
                    new_content.append(item)

            # Reconstruct message (simplified for tracing)
            new_messages.append(
                {"role": getattr(msg, "type", "user"), "content": new_content}
            )
        else:
            new_messages.append(msg)

    return {**inputs, "messages": new_messages}
    
    ...
    llm = ChatBedrockConverse(
                model_id=settings.BEDROCK_MODEL_ID,
                region_name=settings.AWS_REGION,
            )
    # Bind the Pydantic model for structured output
    self._llm_with_structure = llm.with_structured_output(PydanticClass)
    
    ... 
    # inside the function where I call Bedrock 
    ls_client = Client(hide_inputs=hide_images_from_langsmith)

    try:
        content_payload = []

        # 1. Prepare images for multimodal input
        for img_bytes in image_bytes_list:
            b64_img = base64.b64encode(img_bytes).decode("utf-8")
            content_payload.append(
                {
                    "type": "image",
                    "source": {
                        "type": "base64",
                        "media_type": "image/jpeg",
                        "data": b64_img,
                    },
                }
            )

        # 2. Add text prompt
        content_payload.append({"type": "text", "text": prompt})

        messages = [
            (
                "system",
                "You are an expert data extraction assistant. Output valid JSON only.",
            ),
            ("human", content_payload),
        ]

        # 3. Pass the custom client via langsmith_extra in the config
        config: RunnableConfig = {
            "metadata": {"source": "FILE_NAME"},
            "langsmith_extra": {"client": ls_client},
        }

        # Invoke LLM with custom config
        data = self.llm_with_structure.invoke(messages, config=config)
    ```