fix: fix OpenAIChatGenerator response_format serialization errors (#9858)

* fix: fix OpenAIChatGenerator response_format serialization errors

* relnote fix

haystack/components/generators/chat/openai.py

@@ -220,7 +220,7 @@ class OpenAIChatGenerator:
 
         # If the response format is a Pydantic model, it's converted to openai's json schema format
         # If it's already a json schema, it's left as is
-        if response_format and issubclass(response_format, BaseModel):
+        if response_format and isinstance(response_format, type) and issubclass(response_format, BaseModel):
             json_schema = {
                 "type": "json_schema",
                 "json_schema": {

releasenotes/notes/fix-openaichat-serialization-json-object-925de0a6387ff115.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Ensure that the `OpenAIChatGenerator` is properly serialized when `response_format` in `generation_kwargs` is
+    provided as a dictionary (for example, `{"type": "json_object"}`). Previously, this caused serialization errors.

test/components/generators/chat/test_openai.py

@@ -341,6 +341,31 @@ class TestOpenAIChatGenerator:
             },
         }
 
+    def test_to_dict_with_response_format_json_object(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = OpenAIChatGenerator(
+            api_key=Secret.from_env_var("OPENAI_API_KEY"),
+            model="gpt-4o-mini",
+            generation_kwargs={"response_format": {"type": "json_object"}},
+        )
+        data = component.to_dict()
+        assert data == {
+            "type": "haystack.components.generators.chat.openai.OpenAIChatGenerator",
+            "init_parameters": {
+                "api_key": {"env_vars": ["OPENAI_API_KEY"], "strict": True, "type": "env_var"},
+                "model": "gpt-4o-mini",
+                "api_base_url": None,
+                "organization": None,
+                "streaming_callback": None,
+                "generation_kwargs": {"response_format": {"type": "json_object"}},
+                "tools": None,
+                "tools_strict": False,
+                "max_retries": None,
+                "timeout": None,
+                "http_client_kwargs": None,
+            },
+        }
+
     def test_from_dict(self, monkeypatch):
         monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key")
         data = {