diff --git a/haystack/dataclasses/chat_message.py b/haystack/dataclasses/chat_message.py index 4cd30c8b74c..77d4b2a30ca 100644 --- a/haystack/dataclasses/chat_message.py +++ b/haystack/dataclasses/chat_message.py @@ -775,6 +775,59 @@ def _validate_openai_message(message: dict[str, Any]) -> None: elif not content: raise ValueError(f"The `content` field is required for {role} messages.") + @staticmethod + def _parse_openai_data_url(data_url: str) -> tuple[str | None, str]: + """ + Split a base64 data URL in OpenAI format into its MIME type and base64 payload. + + :param data_url: A data URL in the format `data:;base64,`. + :returns: A tuple containing the MIME type (or None if absent) and the base64 data. + :raises ValueError: If the URL is not a base64 data URL. + """ + if not data_url.startswith("data:") or ";base64," not in data_url: + raise ValueError( + f"Unsupported URL: {data_url!r}. Only base64 data URLs in the format " + "`data:;base64,` are supported." + ) + header, base64_data = data_url.split(";base64,", 1) + return header[len("data:") :] or None, base64_data + + @classmethod + def _from_openai_content_parts(cls, content: list[Any]) -> list[TextContent | ImageContent | FileContent]: + """ + Convert a list of content parts in OpenAI format into Haystack content parts. + + :param content: A list of content parts in OpenAI format. + :returns: A list of TextContent, ImageContent, and FileContent objects. + :raises ValueError: If a content part is malformed or of an unsupported type. + """ + parts: list[TextContent | ImageContent | FileContent] = [] + for part in content: + part_type = part.get("type") if isinstance(part, dict) else None + if part_type == "text": + parts.append(TextContent(text=part["text"])) + elif part_type == "image_url": + image_url = part.get("image_url") or {} + mime_type, base64_image = cls._parse_openai_data_url(image_url.get("url", "")) + parts.append( + ImageContent(base64_image=base64_image, mime_type=mime_type, detail=image_url.get("detail")) + ) + elif part_type == "file": + file = part.get("file") or {} + file_data = file.get("file_data") + if not file_data: + raise ValueError( + f"Unsupported file content part: {part}. Only files with inline base64 `file_data` are " + "supported: files referenced by `file_id` cannot be converted." + ) + mime_type, base64_data = cls._parse_openai_data_url(file_data) + parts.append(FileContent(base64_data=base64_data, mime_type=mime_type, filename=file.get("filename"))) + else: + raise ValueError( + f"Unsupported content part: {part}. Supported part types are `text`, `image_url`, and `file`." + ) + return parts + @classmethod def from_openai_dict_format(cls, message: dict[str, Any]) -> "ChatMessage": """ @@ -791,7 +844,7 @@ def from_openai_dict_format(cls, message: dict[str, Any]) -> "ChatMessage": The created ChatMessage object. :raises ValueError: - If the message dictionary is missing required fields. + If the message dictionary is missing required fields or contains unsupported content parts. """ cls._validate_openai_message(message) @@ -820,9 +873,21 @@ def from_openai_dict_format(cls, message: dict[str, Any]) -> "ChatMessage": assert content is not None # ensured by _validate_openai_message, but we need to make mypy happy if role == "user": - return cls.from_user(text=content, name=name) + if isinstance(content, str): + return cls.from_user(text=content, name=name) + return cls.from_user(content_parts=cls._from_openai_content_parts(content), name=name) if role in ["system", "developer"]: - return cls.from_system(text=content, name=name) + if isinstance(content, str): + return cls.from_system(text=content, name=name) + # OpenAI only supports text content parts for system and developer messages + texts = [] + for part in content: + if not isinstance(part, dict) or part.get("type") != "text": + raise ValueError( + f"Unsupported content part in {role} message: {part}. Only text parts are supported." + ) + texts.append(part["text"]) + return cls.from_system(text="\n".join(texts), name=name) if isinstance(content, list): if not all("text" in el for el in content): diff --git a/releasenotes/notes/fix-chat-message-openai-list-content-8a4b76b4a3b1885a.yaml b/releasenotes/notes/fix-chat-message-openai-list-content-8a4b76b4a3b1885a.yaml new file mode 100644 index 00000000000..6f1075ad1e1 --- /dev/null +++ b/releasenotes/notes/fix-chat-message-openai-list-content-8a4b76b4a3b1885a.yaml @@ -0,0 +1,10 @@ +--- +fixes: + - | + Fixed ``ChatMessage.from_openai_dict_format`` handling of messages whose ``content`` is a list of content parts. + Previously, the list was wrapped verbatim in a single ``TextContent``, so ``msg.text`` returned a list instead of + a string and re-serializing the message produced invalid OpenAI content. User messages now correctly convert + ``text`` parts to ``TextContent``, ``image_url`` parts with base64 data URLs to ``ImageContent``, and ``file`` + parts with inline ``file_data`` to ``FileContent``. System and developer messages accept lists of ``text`` parts. + Unsupported content parts (for example, non-data image URLs or files referenced by ``file_id``) now raise a + ``ValueError`` instead of silently corrupting the message. diff --git a/test/dataclasses/test_chat_message.py b/test/dataclasses/test_chat_message.py index 483a28cbd16..e3b8612565a 100644 --- a/test/dataclasses/test_chat_message.py +++ b/test/dataclasses/test_chat_message.py @@ -1024,6 +1024,98 @@ def test_from_openai_dict_format_system_message(self): assert message.role.value == "system" assert message.text == "You are a helpful assistant" + def test_from_openai_dict_format_user_message_with_text_parts(self): + openai_msg = { + "role": "user", + "content": [{"type": "text", "text": "part one"}, {"type": "text", "text": "part two"}], + } + message = ChatMessage.from_openai_dict_format(openai_msg) + assert message.role.value == "user" + assert message.text == "part one" + assert message.texts == ["part one", "part two"] + + def test_from_openai_dict_format_user_message_with_image_part(self, base64_image_string): + openai_msg = { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{base64_image_string}", "detail": "low"}, + }, + ], + } + message = ChatMessage.from_openai_dict_format(openai_msg) + assert message.role.value == "user" + assert message.text == "What is in this image?" + assert message.images == [ImageContent(base64_image=base64_image_string, mime_type="image/png", detail="low")] + + def test_from_openai_dict_format_user_message_with_file_part(self, base64_pdf_string): + openai_msg = { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this document"}, + { + "type": "file", + "file": {"file_data": f"data:application/pdf;base64,{base64_pdf_string}", "filename": "test.pdf"}, + }, + ], + } + message = ChatMessage.from_openai_dict_format(openai_msg) + assert message.role.value == "user" + assert message.text == "Summarize this document" + assert message.files == [ + FileContent(base64_data=base64_pdf_string, mime_type="application/pdf", filename="test.pdf") + ] + + def test_from_openai_dict_format_user_message_with_unsupported_parts(self, base64_image_string): + # non-data image URLs cannot be converted to ImageContent + with pytest.raises(ValueError): + ChatMessage.from_openai_dict_format( + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://example.com/image.png"}}], + } + ) + # files referenced by file_id cannot be converted to FileContent + with pytest.raises(ValueError): + ChatMessage.from_openai_dict_format( + {"role": "user", "content": [{"type": "file", "file": {"file_id": "file-abc123"}}]} + ) + # unknown content part types are rejected + with pytest.raises(ValueError): + ChatMessage.from_openai_dict_format( + {"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": base64_image_string}}]} + ) + + def test_from_openai_dict_format_system_message_with_text_parts(self): + openai_msg = { + "role": "system", + "content": [{"type": "text", "text": "You are a helpful assistant"}, {"type": "text", "text": "Be brief"}], + } + message = ChatMessage.from_openai_dict_format(openai_msg) + assert message.role.value == "system" + assert message.text == "You are a helpful assistant\nBe brief" + + def test_from_openai_dict_format_system_message_with_non_text_parts(self, base64_image_string): + openai_msg = { + "role": "system", + "content": [{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{base64_image_string}"}}], + } + with pytest.raises(ValueError): + ChatMessage.from_openai_dict_format(openai_msg) + + def test_from_openai_dict_format_multimodal_user_message_round_trip(self, base64_image_string, base64_pdf_string): + message = ChatMessage.from_user( + content_parts=[ + TextContent(text="Compare this image and document"), + ImageContent(base64_image=base64_image_string, mime_type="image/png", detail="high"), + FileContent(base64_data=base64_pdf_string, mime_type="application/pdf", filename="test.pdf"), + ] + ) + round_tripped = ChatMessage.from_openai_dict_format(message.to_openai_dict_format()) + assert round_tripped == message + def test_from_openai_dict_format_assistant_message_with_content(self): openai_msg = {"role": "assistant", "content": "I can help with that"} message = ChatMessage.from_openai_dict_format(openai_msg)