fix(qwen): use DashScope native image format for Qwen vision models

sheyanmin · sheyanmin · commit fd4e295390e4 · 2026-06-13T22:22:07.000+08:00
DashScope's OpenAI-compatible endpoint rejects the standard `image_url` content part type with 'Unexpected item type in content' for Qwen vision models. Convert to DashScope native format (`type: image` with direct data URI string) when the provider or baseUrl indicates a DashScope endpoint. Detection: provider includes 'dashscope', provider is 'qwen' or 'qwen-dashscope', or baseUrl includes 'dashscope.aliyuncs.com'. Closes #92688
diff --git a/src/llm/providers/openai-completions.ts b/src/llm/providers/openai-completions.ts
@@ -81,6 +81,22 @@ function isImageContentBlock(block: { type: string }): block is ImageContent {
   return block.type === "image";
 }
 
+function isDashScopeEndpoint(model: Model<"openai-completions">): boolean {
+  const provider = model.provider?.toLowerCase() ?? "";
+  const baseUrl = model.baseUrl?.toLowerCase() ?? "";
+  return (
+    provider.includes("dashscope") ||
+    provider === "qwen" ||
+    provider === "qwen-dashscope" ||
+    baseUrl.includes("dashscope.aliyuncs.com")
+  );
+}
+
+type DashScopeImageContentPart = {
+  type: "image";
+  image: string;
+};
+
 export interface OpenAICompletionsOptions extends StreamOptions {
   toolChoice?: "auto" | "none" | "required" | { type: "function"; function: { name: string } };
   reasoningEffort?: "minimal" | "low" | "medium" | "high" | "xhigh";
@@ -950,6 +966,7 @@ export function convertMessages(
           content: sanitizeSurrogates(msg.content),
         });
       } else {
+        const useDashScopeFormat = isDashScopeEndpoint(model);
         const content: ChatCompletionContentPart[] = msg.content.map(
           (item): ChatCompletionContentPart => {
             if (item.type === "text") {
@@ -958,6 +975,12 @@ export function convertMessages(
                 text: sanitizeSurrogates(item.text),
               } satisfies ChatCompletionContentPartText;
             }
+            if (useDashScopeFormat) {
+              return {
+                type: "image",
+                image: `data:${item.mimeType};base64,${item.data}`,
+              } as unknown as ChatCompletionContentPart;
+            }
             return {
               type: "image_url",
               image_url: {
@@ -1079,7 +1102,10 @@ export function convertMessages(
       }
       params.push(assistantMsg);
     } else if (msg.role === "toolResult") {
-      const imageBlocks: Array<{ type: "image_url"; image_url: { url: string } }> = [];
+      const useDashScopeFormat = isDashScopeEndpoint(model);
+      const imageBlocks: Array<
+        { type: "image_url"; image_url: { url: string } } | { type: "image"; image: string }
+      > = [];
       let j = i;
 
       for (; j < transformedMessages.length && transformedMessages[j].role === "toolResult"; j++) {
@@ -1108,12 +1134,19 @@ export function convertMessages(
         if (hasImages && model.input.includes("image")) {
           for (const block of toolMsg.content) {
             if (isImageContentBlock(block)) {
-              imageBlocks.push({
-                type: "image_url",
-                image_url: {
-                  url: `data:${block.mimeType};base64,${block.data}`,
-                },
-              });
+              if (useDashScopeFormat) {
+                imageBlocks.push({
+                  type: "image",
+                  image: `data:${block.mimeType};base64,${block.data}`,
+                });
+              } else {
+                imageBlocks.push({
+                  type: "image_url",
+                  image_url: {
+                    url: `data:${block.mimeType};base64,${block.data}`,
+                  },
+                });
+              }
             }
           }
         }
@@ -1137,8 +1170,8 @@ export function convertMessages(
               text: "Attached image(s) from tool result:",
             },
             ...imageBlocks,
-          ],
-        });
+          ] as ChatCompletionContentPart[],
+        } as ChatCompletionMessageParam);
         lastRole = "user";
       } else {
         lastRole = "toolResult";
