From 40775bdd6ab165288d332cf27cf918960ea79f49 Mon Sep 17 00:00:00 2001 From: Bendr0id Date: Wed, 2 Sep 2026 16:51:45 +0200 Subject: [PATCH] server: make OpenAI-compatible image input work end to end Analyzing an image over the OpenAI chat completions API failed even though /read in the agent handled the same file, and every failure surfaced as a generic "invalid JSON request". Three independent causes, each sufficient on its own to break it: The data URI parser matched a fixed set of literal prefixes and the base64 decoder accepted only canonical input, so payloads real clients emit were refused: varying scheme and media-type case, parameters such as ";charset=" before ";base64", MIME line breaks, the URL-safe alphabet and omitted padding. Rejections now explain themselves instead of collapsing into one opaque 400, and an untyped content block carrying an image_url is treated as an image rather than silently dropped, which had answered the prompt from the text alone as if no image had been sent. /v1/models never advertised image input, so clients that gate on modality stripped the image before it left the client. Model metadata now carries architecture.input_modalities when an encoder is loaded. Clients also commonly suppress image input for anything they classify as DeepSeek, because the upstream API is text-only, and re-enable it only when the model id carries a "vision" token, so deepseek-v4-flash-vision and deepseek-v4-pro-vision are offered too. Both are reported only when started with --vision, including the per-model endpoint. An image the server cannot decode is replayed in the transcript on every later turn, so a single WebP wedged a conversation permanently: every request failed regardless of what was actually being asked. An undecodable image now becomes a text note in the history and only the newest message rejects the request, since that is the one the caller just attached and needs to be told about. Dropping an image also removes the marker that referenced it, which rendering would otherwise emit into the prompt as literal control-character junk. --- ds4_server.c | 709 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 616 insertions(+), 93 deletions(-) diff --git a/ds4_server.c b/ds4_server.c index 50efbe6bc..75680f1ad 100644 --- a/ds4_server.c +++ b/ds4_server.c @@ -48,8 +48,11 @@ static volatile sig_atomic_t g_listen_fd = -1; #if defined(__GNUC__) || defined(__clang__) #define DS4_SERVER_MAYBE_UNUSED __attribute__((unused)) +#define DS4_SERVER_PRINTF(fmt, args) \ + __attribute__((format(printf, fmt, args))) #else #define DS4_SERVER_MAYBE_UNUSED +#define DS4_SERVER_PRINTF(fmt, args) #endif static void stop_signal_handler(int sig) { @@ -442,10 +445,17 @@ typedef struct { size_t encoded_len; } server_image_input; +#define SERVER_IMAGE_ERROR_BYTES 224 + typedef struct { server_image_input *v; size_t len; size_t cap; + /* Why the last rejected image block was refused. Image blocks are parsed + * deep inside the JSON walkers, which can only report a boolean, so the + * request parsers would otherwise collapse every bad image into a generic + * "invalid JSON request" and leave the client with nothing to act on. */ + char error[SERVER_IMAGE_ERROR_BYTES]; } server_image_inputs; static void server_image_inputs_free(server_image_inputs *images) { @@ -455,60 +465,125 @@ static void server_image_inputs_free(server_image_inputs *images) { memset(images, 0, sizeof(*images)); } +DS4_SERVER_PRINTF(2, 3) +static void server_image_inputs_error(server_image_inputs *images, + const char *fmt, ...) { + if (!images || images->error[0]) return; + va_list ap; + va_start(ap, fmt); + vsnprintf(images->error, sizeof(images->error), fmt, ap); + va_end(ap); +} + static int base64_value(unsigned char c) { if (c >= 'A' && c <= 'Z') return c - 'A'; if (c >= 'a' && c <= 'z') return 26 + c - 'a'; if (c >= '0' && c <= '9') return 52 + c - '0'; - if (c == '+') return 62; - if (c == '/') return 63; + /* Accept the URL-safe alphabet too: '-' and '_' never appear in standard + * base64, so this cannot change how valid standard input decodes. */ + if (c == '+' || c == '-') return 62; + if (c == '/' || c == '_') return 63; return -1; } +/* Decode base64 the way real clients emit it: MIME line breaks, indentation, + * the URL-safe alphabet and omitted '=' padding are all tolerated. */ static bool server_decode_base64(const char *src, uint8_t **out, size_t *out_len) { - const size_t n = src ? strlen(src) : 0; - if (n == 0 || (n & 3u) != 0 || n > 64u * 1024u * 1024u) - return false; - size_t cap = (n / 4u) * 3u; - uint8_t *decoded = xmalloc(cap ? cap : 1); + if (!src) return false; + const size_t n = strlen(src); + if (n == 0 || n > 96u * 1024u * 1024u) return false; + uint8_t *decoded = xmalloc((n / 4u + 1u) * 3u + 4u); size_t used = 0; - for (size_t i = 0; i < n; i += 4) { - int a = base64_value((unsigned char)src[i]); - int b = base64_value((unsigned char)src[i + 1]); - bool pad2 = src[i + 2] == '='; - bool pad3 = src[i + 3] == '='; - int c = pad2 ? 0 : base64_value((unsigned char)src[i + 2]); - int d = pad3 ? 0 : base64_value((unsigned char)src[i + 3]); - if (a < 0 || b < 0 || c < 0 || d < 0 || - (pad2 && !pad3) || ((pad2 || pad3) && i + 4 != n)) { + size_t symbols = 0; + uint32_t acc = 0; + unsigned bits = 0; + bool padded = false; + for (size_t i = 0; i < n; i++) { + const unsigned char c = (unsigned char)src[i]; + if (isspace(c)) continue; + if (c == '=') { + padded = true; + continue; + } + const int v = base64_value(c); + if (v < 0 || padded) { free(decoded); return false; } - uint32_t bits = ((uint32_t)a << 18) | ((uint32_t)b << 12) | - ((uint32_t)c << 6) | (uint32_t)d; - decoded[used++] = (uint8_t)(bits >> 16); - if (!pad2) decoded[used++] = (uint8_t)(bits >> 8); - if (!pad3) decoded[used++] = (uint8_t)bits; + acc = (acc << 6) | (uint32_t)v; + bits += 6; + symbols++; + if (bits >= 8) { + bits -= 8; + decoded[used++] = (uint8_t)(acc >> bits); + } + } + /* A single trailing symbol encodes no whole byte and cannot occur in a + * complete stream, so treat it as corruption rather than silently + * truncating. */ + if (used == 0 || (symbols & 3u) == 1u) { + free(decoded); + return false; } *out = decoded; *out_len = used; return true; } -static bool server_image_media_type(const char *media_type) { - return media_type && - (!strcasecmp(media_type, "image/png") || - !strcasecmp(media_type, "image/jpeg") || - !strcasecmp(media_type, "image/jpg")); +/* The image decoder dispatches on magic bytes and ignores any declared media + * type, so the media type is only used for diagnostics. */ +static const char *server_image_sniff_format(const uint8_t *b, size_t n) { + if (n >= 8 && !memcmp(b, "\x89PNG\r\n\x1a\n", 8)) return "PNG"; + if (n >= 3 && b[0] == 0xff && b[1] == 0xd8 && b[2] == 0xff) return "JPEG"; + if (n >= 12 && !memcmp(b, "RIFF", 4) && !memcmp(b + 8, "WEBP", 4)) + return "WebP"; + if (n >= 6 && (!memcmp(b, "GIF87a", 6) || !memcmp(b, "GIF89a", 6))) + return "GIF"; + if (n >= 12 && (!memcmp(b + 4, "ftypheic", 8) || + !memcmp(b + 4, "ftypheix", 8) || + !memcmp(b + 4, "ftypmif1", 8) || + !memcmp(b + 4, "ftypavif", 8))) + return "HEIC/AVIF"; + if (n >= 2 && b[0] == 'B' && b[1] == 'M') return "BMP"; + if (n >= 4 && !memcmp(b, "II*\0", 4)) return "TIFF"; + if (n >= 4 && !memcmp(b, "MM\0*", 4)) return "TIFF"; + return NULL; } static bool server_image_inputs_push_base64(server_image_inputs *images, const char *media_type, const char *base64, char marker[SERVER_IMAGE_MARKER_BYTES]) { - if (!server_image_media_type(media_type)) return false; + /* Only the family is checked: a client that mislabels a JPEG as + * "image/png" still works because decoding sniffs the real format. */ + if (media_type && media_type[0] && + strncasecmp(media_type, "image/", 6) != 0 && + strcasecmp(media_type, "application/octet-stream") != 0) { + server_image_inputs_error(images, + "image media type \"%s\" is not an image; " + "send a JPEG or PNG", media_type); + return false; + } server_image_input image = {0}; - if (!server_decode_base64(base64, &image.encoded, &image.encoded_len)) + if (!server_decode_base64(base64, &image.encoded, &image.encoded_len)) { + server_image_inputs_error(images, + "image data is not valid base64"); return false; + } + const char *format = + server_image_sniff_format(image.encoded, image.encoded_len); + if (!format || (strcmp(format, "PNG") && strcmp(format, "JPEG"))) { + if (format) + server_image_inputs_error(images, + "%s images are not supported; " + "send a JPEG or PNG", format); + else + server_image_inputs_error(images, + "image data is not a JPEG or PNG " + "(decoded %zu bytes)", image.encoded_len); + free(image.encoded); + return false; + } unsigned char nonce[12]; if (!random_bytes(nonce, sizeof(nonce))) { uint64_t fallback = (uint64_t)time(NULL) ^ @@ -536,23 +611,61 @@ static bool server_image_inputs_push_base64(server_image_inputs *images, return true; } +/* Parse an RFC 2397 data URI: "data:" [media-type] *(";" parameter) ";base64" + * "," payload. Matching a fixed set of literal prefixes is not enough, + * because clients legitimately vary the case of the scheme and media type and + * insert parameters such as ";charset=utf-8" before ";base64". */ static bool server_image_inputs_push_data_uri( server_image_inputs *images, const char *uri, char marker[SERVER_IMAGE_MARKER_BYTES]) { - static const char png[] = "data:image/png;base64,"; - static const char jpeg[] = "data:image/jpeg;base64,"; - static const char jpg[] = "data:image/jpg;base64,"; - if (!uri) return false; - if (!strncmp(uri, png, sizeof(png) - 1)) - return server_image_inputs_push_base64( - images, "image/png", uri + sizeof(png) - 1, marker); - if (!strncmp(uri, jpeg, sizeof(jpeg) - 1)) - return server_image_inputs_push_base64( - images, "image/jpeg", uri + sizeof(jpeg) - 1, marker); - if (!strncmp(uri, jpg, sizeof(jpg) - 1)) - return server_image_inputs_push_base64( - images, "image/jpg", uri + sizeof(jpg) - 1, marker); - return false; + if (!uri || !uri[0]) { + server_image_inputs_error(images, "image_url is missing a url"); + return false; + } + if (strncasecmp(uri, "data:", 5) != 0) { + /* Fetching remote URLs would let a request drive outbound traffic from + * the server, and reading local paths would expose the host + * filesystem, so the client must inline the bytes. */ + server_image_inputs_error( + images, + "image_url must be an inline base64 data URI such as " + "\"data:image/jpeg;base64,...\"; the server does not fetch " + "remote URLs or read local file paths"); + return false; + } + + const char *meta = uri + 5; + const char *comma = strchr(meta, ','); + if (!comma) { + server_image_inputs_error(images, + "image data URI is missing the \",\" that " + "separates the header from the payload"); + return false; + } + + char media_type[128] = {0}; + bool base64 = false; + for (const char *seg = meta; seg < comma;) { + const char *end = memchr(seg, ';', (size_t)(comma - seg)); + if (!end || end > comma) end = comma; + const size_t len = (size_t)(end - seg); + if (seg == meta) { + if (len && len < sizeof(media_type)) memcpy(media_type, seg, len); + } else if (len == 6 && !strncasecmp(seg, "base64", 6)) { + base64 = true; + } + seg = (end < comma) ? end + 1 : comma; + } + + if (!base64) { + server_image_inputs_error( + images, + "image data URI must be base64 encoded (expected \";base64,\" " + "before the payload)"); + return false; + } + return server_image_inputs_push_base64(images, media_type, comma + 1, + marker); } static void append_owned_text(char **dst, const char *text) { @@ -563,6 +676,56 @@ static void append_owned_text(char **dst, const char *text) { *dst = buf_take(&b); } +/* Stand in for an image that could not be decoded, so the model is told the + * image existed instead of answering from the surrounding text as if it had + * never been sent. */ +#define SERVER_IMAGE_NOTE_BYTES (SERVER_IMAGE_ERROR_BYTES + 24) + +static void format_dropped_image_note(char note[SERVER_IMAGE_NOTE_BYTES], + const char *why) { + snprintf(note, SERVER_IMAGE_NOTE_BYTES, "[image omitted: %s]", + why && why[0] ? why : "not a supported image"); +} + +static void append_dropped_image_note(char **dst, const char *why) { + char note[SERVER_IMAGE_NOTE_BYTES]; + format_dropped_image_note(note, why); + append_owned_text(dst, note); +} + +static void text_remove_all(char **s, const char *needle) { + if (!s || !*s || !needle || !needle[0]) return; + const size_t n = strlen(needle); + char *src = *s; + buf b = {0}; + for (const char *q = src;;) { + const char *hit = strstr(q, needle); + if (!hit) { + buf_puts(&b, q); + break; + } + buf_append(&b, q, (size_t)(hit - q)); + q = hit + n; + } + free(src); + *s = buf_take(&b); +} + +/* Discard decoded images and the markers standing in for them, keeping the + * diagnostic that rejected them. Rendering pairs every image with its marker + * in the prompt text, so a marker left behind would be tokenized as literal + * junk. */ +static void server_image_inputs_drop(server_image_inputs *images, + char **content) { + if (!images) return; + for (size_t i = 0; i < images->len; i++) + text_remove_all(content, images->v[i].marker); + char saved[SERVER_IMAGE_ERROR_BYTES]; + snprintf(saved, sizeof(saved), "%s", images->error); + server_image_inputs_free(images); + snprintf(images->error, sizeof(images->error), "%s", saved); +} + static bool json_content(const char **p, char **out) { json_ws(p); if (**p == '"') return json_string(p, out); @@ -752,8 +915,25 @@ typedef struct { chat_msg *v; int len; int cap; + /* Diagnostic for a rejected content block whose owning message was + * discarded before the request-level error was formatted. */ + char image_error[SERVER_IMAGE_ERROR_BYTES]; } chat_msgs; +/* Undecodable images are tolerated in the transcript but not in the turn being + * answered. A client replays the whole conversation on every request, so one + * stale image the server cannot read would otherwise reject every later + * request forever; an image the caller just attached still has to be reported + * rather than quietly answered from its surrounding text. */ +static bool chat_msgs_newest_image_rejected(chat_msgs *msgs) { + if (!msgs || msgs->len <= 0) return false; + const chat_msg *last = &msgs->v[msgs->len - 1]; + if (!last->images.error[0]) return false; + snprintf(msgs->image_error, sizeof(msgs->image_error), "%s", + last->images.error); + return true; +} + static void tool_memory_attach_to_messages(server *s, chat_msgs *msgs, tool_replay_stats *stats); static bool tool_memory_has_id(server *s, const char *id); @@ -889,6 +1069,24 @@ static void chat_msgs_free(chat_msgs *msgs) { memset(msgs, 0, sizeof(*msgs)); } +/* Report why a request was rejected, preferring a specific image diagnostic + * over the generic parse failure. Must be called before the messages are + * freed. */ +static void chat_msgs_request_error(const chat_msgs *msgs, char *err, + size_t errlen) { + if (msgs && msgs->image_error[0]) { + snprintf(err, errlen, "%s", msgs->image_error); + return; + } + for (int i = 0; msgs && i < msgs->len; i++) { + if (msgs->v[i].images.error[0]) { + snprintf(err, errlen, "%s", msgs->v[i].images.error); + return; + } + } + snprintf(err, errlen, "invalid JSON request"); +} + static void chat_msgs_push(chat_msgs *msgs, chat_msg msg) { if (msgs->len == msgs->cap) { msgs->cap = msgs->cap ? msgs->cap * 2 : 8; @@ -1139,7 +1337,13 @@ static const char *server_model_id_from_engine(ds4_engine *engine) { "deepseek-v4-pro" : "deepseek-v4-flash"; } -static bool server_model_alias_known(const char *id) { +/* The vision-tagged aliases exist only to stop clients from stripping image + * input, so they must not appear unless an encoder is actually loaded: + * advertising them on a text-only server would invite images it cannot read. */ +static bool server_model_alias_known(const char *id, bool vision) { + if (id && (!strcmp(id, "deepseek-v4-flash-vision") || + !strcmp(id, "deepseek-v4-pro-vision"))) + return vision; return id && (!strcmp(id, "deepseek-v4-flash") || !strcmp(id, "deepseek-v4-pro") || @@ -1911,11 +2115,24 @@ static bool parse_openai_content_object(const char **p, chat_msg *msg) { if (**p != '}') goto bad; (*p)++; - if (type && (!strcmp(type, "image_url") || !strcmp(type, "input_image"))) { + /* An untyped block carrying an image_url is an image: dropping it would + * answer the prompt from the text alone as if no image had been sent. */ + const bool is_image = image_url && (!type || + !strcmp(type, "image_url") || + !strcmp(type, "input_image") || + !strcmp(type, "image")); + if (is_image) { char marker[SERVER_IMAGE_MARKER_BYTES]; - if (!server_image_inputs_push_data_uri(&msg->images, image_url, marker)) - goto bad; - append_owned_text(&msg->content, marker); + if (server_image_inputs_push_data_uri(&msg->images, image_url, marker)) + append_owned_text(&msg->content, marker); + else + append_dropped_image_note(&msg->content, msg->images.error); + } else if (type && (!strcmp(type, "image_url") || + !strcmp(type, "input_image"))) { + server_image_inputs_error(&msg->images, + "content block of type \"%s\" has no " + "image_url", type); + append_dropped_image_note(&msg->content, msg->images.error); } else if (text) { append_owned_text(&msg->content, text); } @@ -2028,7 +2245,13 @@ static bool parse_messages(const char **p, chat_msgs *msgs) { (*p)++; if (!msg.role) msg.role = xstrdup("user"); if (!msg.content) msg.content = xstrdup(""); - if (msg.images.len && strcmp(msg.role, "user")) goto fail; + if (msg.images.len && strcmp(msg.role, "user")) { + server_image_inputs_error(&msg.images, + "images are only accepted on \"user\" " + "messages, not \"%s\"", msg.role); + server_image_inputs_drop(&msg.images, &msg.content); + append_dropped_image_note(&msg.content, msg.images.error); + } chat_msgs_push(msgs, msg); memset(&msg, 0, sizeof(msg)); json_ws(p); @@ -2036,11 +2259,17 @@ static bool parse_messages(const char **p, chat_msgs *msgs) { json_ws(p); continue; fail: + /* The message that owns the diagnostic is about to be freed, so lift + * it to the conversation for the request-level error. */ + if (msg.images.error[0] && !msgs->image_error[0]) + snprintf(msgs->image_error, sizeof(msgs->image_error), "%s", + msg.images.error); chat_msg_free(&msg); return false; } if (**p != ']') return false; (*p)++; + if (chat_msgs_newest_image_rejected(msgs)) return false; return true; } @@ -2193,11 +2422,27 @@ static bool parse_anthropic_content_block(const char **p, const char *role, chat msg->content = buf_take(&b); } else if (type && !strcmp(type, "image")) { char marker[SERVER_IMAGE_MARKER_BYTES]; - if (!source_type || strcmp(source_type, "base64") || - !server_image_inputs_push_base64(&msg->images, media_type, - image_data, marker)) - goto bad; - append_owned_text(&msg->content, marker); + if (source_type && !strcmp(source_type, "url")) { + server_image_inputs_error( + &msg->images, + "image source type \"url\" is not supported; send " + "{\"type\":\"base64\",\"media_type\":...,\"data\":...} " + "because the server does not fetch remote URLs"); + append_dropped_image_note(&msg->content, msg->images.error); + } else if (source_type && strcmp(source_type, "base64")) { + /* "base64" is the only source Anthropic clients send inline, but a + * missing type is common in hand-rolled payloads; the data itself + * is still validated by the push. */ + server_image_inputs_error(&msg->images, + "unsupported image source type \"%s\"", + source_type); + append_dropped_image_note(&msg->content, msg->images.error); + } else if (server_image_inputs_push_base64(&msg->images, media_type, + image_data, marker)) { + append_owned_text(&msg->content, marker); + } else { + append_dropped_image_note(&msg->content, msg->images.error); + } } else { if (text) { buf b = {0}; @@ -2320,7 +2565,13 @@ static bool parse_anthropic_messages(const char **p, chat_msgs *msgs) { (*p)++; if (!msg.role) msg.role = xstrdup("user"); if (!msg.content) msg.content = xstrdup(""); - if (msg.images.len && strcmp(msg.role, "user")) goto fail; + if (msg.images.len && strcmp(msg.role, "user")) { + server_image_inputs_error(&msg.images, + "images are only accepted on \"user\" " + "messages, not \"%s\"", msg.role); + server_image_inputs_drop(&msg.images, &msg.content); + append_dropped_image_note(&msg.content, msg.images.error); + } chat_msgs_push(msgs, msg); memset(&msg, 0, sizeof(msg)); json_ws(p); @@ -2328,11 +2579,17 @@ static bool parse_anthropic_messages(const char **p, chat_msgs *msgs) { json_ws(p); continue; fail: + /* The message that owns the diagnostic is about to be freed, so lift + * it to the conversation for the request-level error. */ + if (msg.images.error[0] && !msgs->image_error[0]) + snprintf(msgs->image_error, sizeof(msgs->image_error), "%s", + msg.images.error); chat_msg_free(&msg); return false; } if (**p != ']') return false; (*p)++; + if (chat_msgs_newest_image_rejected(msgs)) return false; return true; } @@ -3700,9 +3957,9 @@ static bool parse_chat_request(ds4_engine *e, server *s, const char *body, int d free(tool_schemas); return true; bad: + chat_msgs_request_error(&msgs, err, errlen); chat_msgs_free(&msgs); free(tool_schemas); - snprintf(err, errlen, "invalid JSON request"); request_free(r); return false; } @@ -3923,10 +4180,10 @@ static bool parse_anthropic_request(ds4_engine *e, server *s, const char *body, free(tool_schemas); return true; bad: + chat_msgs_request_error(&msgs, err, errlen); chat_msgs_free(&msgs); free(system); free(tool_schemas); - snprintf(err, errlen, "invalid JSON request"); request_free(r); return false; } @@ -4041,17 +4298,21 @@ static bool parse_responses_content_array(const char **p, char **out, !strcmp(type, "text") || !strcmp(type, "summary_text") || !strcmp(type, "reasoning_text")); - bool is_image_block = type && !strcmp(type, "input_image"); + bool is_image_block = image_url && (!type || + !strcmp(type, "input_image") || + !strcmp(type, "image_url") || + !strcmp(type, "image")); if (is_image_block) { char marker[SERVER_IMAGE_MARKER_BYTES]; - if (!images || - !server_image_inputs_push_data_uri(images, image_url, marker)) { - free(type); - free(text); - free(image_url); - goto fail; + if (images && + server_image_inputs_push_data_uri(images, image_url, marker)) { + buf_puts(&b, marker); + } else { + char note[SERVER_IMAGE_NOTE_BYTES]; + format_dropped_image_note(note, images ? images->error : + "images are not accepted here"); + buf_puts(&b, note); } - buf_puts(&b, marker); } else if (!is_text_block || !text) { free(type); free(text); @@ -4094,7 +4355,12 @@ static bool parse_responses_content_array_multimodal( char *tmp = NULL; server_image_inputs_free(images); if (!parse_responses_content_array(p, &tmp, images)) { + /* Freeing clears the diagnostic, so carry it across the reset for the + * caller to turn into a useful 400. */ + char saved[SERVER_IMAGE_ERROR_BYTES]; + snprintf(saved, sizeof(saved), "%s", images ? images->error : ""); server_image_inputs_free(images); + if (images) snprintf(images->error, sizeof(images->error), "%s", saved); return false; } free(*dst); @@ -4171,6 +4437,9 @@ static bool parse_responses_input(const char **p, chat_msgs *msgs, } else if (!strcmp(key, "content")) { if (!parse_responses_content_array_multimodal( p, &content, &content_images)) { + if (content_images.error[0]) + snprintf(msgs->image_error, sizeof(msgs->image_error), + "%s", content_images.error); free(key); goto item_fail; } @@ -4346,24 +4615,11 @@ static bool parse_responses_input(const char **p, chat_msgs *msgs, } if (content_images.len && (strcmp(t, "message") || (role && strcmp(role, "user")))) { - free(type); - free(role); - free(content); - free(name); - free(namespace); - free(call_id); - free(item_id); - free(arguments); - free(output); - free(input_str); - free(summary); - free(action); - free(result); - free(tools_json); - free(status_str); - server_image_inputs_free(&content_images); - buf_free(&pending_reasoning); - return false; + server_image_inputs_error(&content_images, + "images are only accepted on \"user\" " + "messages"); + server_image_inputs_drop(&content_images, &content); + append_dropped_image_note(&content, content_images.error); } /* Three classes of items: * 1. consumes_reasoning: assistant message / function_call / hosted-tool @@ -4593,6 +4849,12 @@ static bool parse_responses_input(const char **p, chat_msgs *msgs, * empty assistant message so the next turn still renders a ... * block. Dropping it loses model state when a previous response ended with * a reasoning-only incomplete turn and the client replays the history. */ + /* Checked before the trailing reasoning item is flushed, so an item that + * carries no content of its own cannot hide a rejected image behind it. */ + if (chat_msgs_newest_image_rejected(msgs)) { + buf_free(&pending_reasoning); + return false; + } if (pending_reasoning.len) { chat_msg msg = {0}; msg.role = xstrdup("assistant"); @@ -4945,11 +5207,11 @@ static bool parse_responses_request(ds4_engine *e, server *s, const char *body, free(tool_schemas); return true; bad: + chat_msgs_request_error(&msgs, err, errlen); chat_msgs_free(&msgs); buf_free(&loaded_tool_schemas); free(instructions); free(tool_schemas); - snprintf(err, errlen, "invalid JSON request"); request_free(r); return false; } @@ -13437,7 +13699,8 @@ typedef struct { } client_arg; static void append_model_json_values(buf *b, const char *id, const char *name, - int ctx, int default_tokens) { + int ctx, int default_tokens, + bool vision) { const int max_completion = default_tokens < ctx ? default_tokens : ctx; buf_printf(b, "{\"id\":"); @@ -13448,6 +13711,23 @@ static void append_model_json_values(buf *b, const char *id, const char *name, "\"owned_by\":\"ds4.c\"," "\"name\":"); json_escape(b, name); + /* Clients decide whether they may send an image from the advertised input + * modalities, and a client that sees text only drops the image before it + * ever reaches the server. Report both the OpenRouter-style nested field + * and a top-level copy, since discovery code in the wild reads either. */ + buf_puts(b, vision + ? "," + "\"architecture\":{" + "\"modality\":\"text+image->text\"," + "\"input_modalities\":[\"text\",\"image\"]," + "\"output_modalities\":[\"text\"]}," + "\"input_modalities\":[\"text\",\"image\"]" + : "," + "\"architecture\":{" + "\"modality\":\"text->text\"," + "\"input_modalities\":[\"text\"]," + "\"output_modalities\":[\"text\"]}," + "\"input_modalities\":[\"text\"]"); buf_printf(b, "," "\"context_length\":%d," @@ -13478,7 +13758,8 @@ static void append_model_json(buf *b, const server *s, const char *id) { id, ds4_engine_model_name(s->engine), s->ctx_size, - s->default_tokens); + s->default_tokens, + ds4_engine_has_vision(s->engine)); } static bool send_model(server *s, int fd, const char *id) { @@ -13500,6 +13781,18 @@ static bool send_models(server *s, int fd) { buf_putc(&b, ','); append_model_json(&b, s, "glm-5.2-reasoner"); } else { + /* Clients commonly suppress image input for every model they + * classify as DeepSeek, because the upstream DeepSeek API is + * text-only, and re-enable it only when the model id carries a + * "vision" token. Offer vision-tagged ids first when an encoder is + * loaded, so such a client will actually send the image instead of + * dropping it and reporting that the model cannot see. */ + if (ds4_engine_has_vision(s->engine)) { + append_model_json(&b, s, "deepseek-v4-flash-vision"); + buf_putc(&b, ','); + append_model_json(&b, s, "deepseek-v4-pro-vision"); + buf_putc(&b, ','); + } append_model_json(&b, s, "deepseek-v4-flash"); buf_putc(&b, ','); append_model_json(&b, s, "deepseek-v4-pro"); @@ -13640,7 +13933,8 @@ static void *client_main(void *arg) { const size_t model_path_prefix_len = strlen(model_path_prefix); if (!strcmp(hr.method, "GET") && !strncmp(hr.path, model_path_prefix, model_path_prefix_len) && - server_model_alias_known(hr.path + model_path_prefix_len)) + server_model_alias_known(hr.path + model_path_prefix_len, + ds4_engine_has_vision(s->engine))) { send_model(s, fd, hr.path + model_path_prefix_len); http_request_free(&hr); @@ -15902,15 +16196,24 @@ static void test_model_alias_thinking_controls(void) { TEST_ASSERT(model_alias_enables_thinking("deepseek-reasoner")); TEST_ASSERT(model_alias_enables_thinking("glm-5.2-reasoner")); TEST_ASSERT(model_alias_enables_thinking("zai/glm-5.2-reasoner")); - TEST_ASSERT(server_model_alias_known("glm-5.2-chat")); - TEST_ASSERT(server_model_alias_known("glm-5.2-reasoner")); + TEST_ASSERT(server_model_alias_known("glm-5.2-chat", false)); + TEST_ASSERT(server_model_alias_known("glm-5.2-reasoner", false)); + /* Vision-tagged ids exist so clients that gate image input on a "vision" + * token in the model id will send images, so they are only known when an + * encoder is loaded. */ + TEST_ASSERT(server_model_alias_known("deepseek-v4-flash-vision", true)); + TEST_ASSERT(server_model_alias_known("deepseek-v4-pro-vision", true)); + TEST_ASSERT(!server_model_alias_known("deepseek-v4-flash-vision", false)); + TEST_ASSERT(!server_model_alias_known("deepseek-v4-pro-vision", false)); + TEST_ASSERT(server_model_alias_known("deepseek-v4-flash", false)); + TEST_ASSERT(!model_alias_disables_thinking("deepseek-v4-flash-vision")); TEST_ASSERT(model_alias_disables_thinking("glm-5.3-flash-chat")); TEST_ASSERT(model_alias_disables_thinking("zai/glm-5.3-flash-chat")); TEST_ASSERT(model_alias_enables_thinking("glm-5.3-flash-reasoner")); TEST_ASSERT(model_alias_enables_thinking("zai/glm-5.3-flash-reasoner")); - TEST_ASSERT(server_model_alias_known("glm-5.3-flash")); - TEST_ASSERT(server_model_alias_known("glm-5.3-flash-chat")); - TEST_ASSERT(server_model_alias_known("glm-5.3-flash-reasoner")); + TEST_ASSERT(server_model_alias_known("glm-5.3-flash", false)); + TEST_ASSERT(server_model_alias_known("glm-5.3-flash-chat", false)); + TEST_ASSERT(server_model_alias_known("glm-5.3-flash-reasoner", false)); } static void test_api_thinking_controls_parse(void) { @@ -17922,7 +18225,7 @@ static void test_tool_history_validation_handles_large_replays(void) { static void test_model_metadata_clamps_completion_to_context(void) { buf b = {0}; append_model_json_values(&b, "deepseek-v4-flash", "DeepSeek V4 Flash", - 32768, 393216); + 32768, 393216, false); TEST_ASSERT(strstr(b.ptr, "\"id\":\"deepseek-v4-flash\"") != NULL); TEST_ASSERT(strstr(b.ptr, "\"name\":\"DeepSeek V4 Flash\"") != NULL); TEST_ASSERT(strstr(b.ptr, "\"context_length\":32768") != NULL); @@ -17931,7 +18234,7 @@ static void test_model_metadata_clamps_completion_to_context(void) { buf_free(&b); append_model_json_values(&b, "deepseek-v4-pro", "DeepSeek V4 Pro", - 100000, 4096); + 100000, 4096, false); TEST_ASSERT(strstr(b.ptr, "\"id\":\"deepseek-v4-pro\"") != NULL); TEST_ASSERT(strstr(b.ptr, "\"name\":\"DeepSeek V4 Pro\"") != NULL); TEST_ASSERT(strstr(b.ptr, "\"context_length\":100000") != NULL); @@ -17939,6 +18242,31 @@ static void test_model_metadata_clamps_completion_to_context(void) { buf_free(&b); } +static void test_model_metadata_advertises_image_input_with_vision(void) { + /* Clients gate image sending on the advertised modalities: OpenAI-style + * agents that read text-only capabilities strip the image from the + * request, so the server never gets a chance to refuse or accept it. */ + buf vision = {0}; + append_model_json_values(&vision, "deepseek-v4-flash", "DeepSeek V4 Flash", + 32768, 4096, true); + TEST_ASSERT(strstr(vision.ptr, + "\"input_modalities\":[\"text\",\"image\"]") != NULL); + TEST_ASSERT(strstr(vision.ptr, "\"architecture\":{") != NULL); + TEST_ASSERT(strstr(vision.ptr, + "\"modality\":\"text+image->text\"") != NULL); + buf_free(&vision); + + /* Without a vision encoder the server must not claim image input, or + * clients will send images that can only be rejected. */ + buf text = {0}; + append_model_json_values(&text, "deepseek-v4-flash", "DeepSeek V4 Flash", + 32768, 4096, false); + TEST_ASSERT(strstr(text.ptr, "\"input_modalities\":[\"text\"]") != NULL); + TEST_ASSERT(strstr(text.ptr, "\"image\"") == NULL); + TEST_ASSERT(strstr(text.ptr, "\"modality\":\"text->text\"") != NULL); + buf_free(&text); +} + static void test_live_prefix_rewind_target(void) { TEST_ASSERT(live_prefix_rewind_target(true, 17, 8, 8) == 7); TEST_ASSERT(live_prefix_rewind_target(true, 49826, 48379, 48379) == 48378); @@ -19351,6 +19679,194 @@ static void test_openai_inline_image_content(void) { buf_free(&json); } +/* Build a single-user-message OpenAI payload whose image_url is VARIANT. */ +static char *test_image_payload(const char *variant) { + buf json = {0}; + buf_puts(&json, + "[{\"role\":\"user\",\"content\":[{\"type\":\"text\"," + "\"text\":\"describe\"},{\"type\":\"image_url\"," + "\"image_url\":{\"url\":\""); + buf_puts(&json, variant); + buf_puts(&json, "\"}}]}]"); + return buf_take(&json); +} + +static void test_openai_image_accepts_client_data_uri_variants(void) { + /* Real OpenAI-compatible clients vary the case of the scheme and media + * type, add parameters before ";base64", wrap the payload at 76 columns + * (base64.encodebytes), drop '=' padding and use the URL-safe alphabet. + * Every one of these used to collapse into "invalid JSON request". */ + buf wrapped = {0}; + for (size_t i = 0; test_inline_png_base64[i]; i++) { + buf_putc(&wrapped, test_inline_png_base64[i]); + if ((i + 1) % 24 == 0) buf_puts(&wrapped, "\n "); + } + buf unpadded = {0}; + for (size_t i = 0; test_inline_png_base64[i]; i++) + if (test_inline_png_base64[i] != '=') + buf_putc(&unpadded, test_inline_png_base64[i]); + + buf variants[6] = {0}; + buf_puts(&variants[0], "data:image/png;base64,"); + buf_puts(&variants[0], test_inline_png_base64); + buf_puts(&variants[1], "DATA:IMAGE/PNG;BASE64,"); + buf_puts(&variants[1], test_inline_png_base64); + buf_puts(&variants[2], "data:image/png;charset=utf-8;base64,"); + buf_puts(&variants[2], test_inline_png_base64); + buf_puts(&variants[3], "data:image/png;base64,"); + buf_puts(&variants[3], wrapped.ptr); + buf_puts(&variants[4], "data:image/png;base64,"); + buf_puts(&variants[4], unpadded.ptr); + /* Mislabelled media type: decoding sniffs magic bytes, so a PNG sent as + * image/jpeg must still work rather than being refused on the label. */ + buf_puts(&variants[5], "data:image/jpeg;base64,"); + buf_puts(&variants[5], test_inline_png_base64); + + for (size_t i = 0; i < sizeof(variants) / sizeof(variants[0]); i++) { + char *payload = test_image_payload(variants[i].ptr); + const char *p = payload; + chat_msgs msgs = {0}; + TEST_ASSERT(parse_messages(&p, &msgs)); + TEST_ASSERT(msgs.len == 1); + TEST_ASSERT(msgs.v[0].images.len == 1); + TEST_ASSERT(msgs.v[0].images.v[0].encoded_len >= 8); + TEST_ASSERT(!memcmp(msgs.v[0].images.v[0].encoded, + "\x89PNG\r\n\x1a\n", 8)); + chat_msgs_free(&msgs); + free(payload); + buf_free(&variants[i]); + } + buf_free(&wrapped); + buf_free(&unpadded); +} + +static void test_openai_image_block_without_type_is_not_dropped(void) { + /* An untyped block carrying an image_url must be treated as an image; + * ignoring it would answer from the text alone as if no image was sent. */ + buf json = {0}; + buf_puts(&json, + "[{\"role\":\"user\",\"content\":[{\"image_url\":{\"url\":" + "\"data:image/png;base64,"); + buf_puts(&json, test_inline_png_base64); + buf_puts(&json, "\"}}]}]"); + const char *p = json.ptr; + chat_msgs msgs = {0}; + TEST_ASSERT(parse_messages(&p, &msgs)); + TEST_ASSERT(msgs.len == 1); + TEST_ASSERT(msgs.v[0].images.len == 1); + chat_msgs_free(&msgs); + buf_free(&json); +} + +static void test_rejected_image_reports_actionable_error(void) { + /* A rejected image must explain itself instead of surfacing the generic + * "invalid JSON request", which leaves the client with nothing to fix. */ + struct { + const char *url; + const char *needle; + } cases[] = { + {"https://example.com/a.png", "does not fetch remote URLs"}, + {"/tmp/a.png", "does not fetch remote URLs"}, + {"data:image/png,notbase64", "base64"}, + {"data:image/png;base64,!!!!not base64!!!!", "not valid base64"}, + {"data:text/plain;base64,aGVsbG8=", "not an image"}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + char *payload = test_image_payload(cases[i].url); + const char *p = payload; + chat_msgs msgs = {0}; + TEST_ASSERT(!parse_messages(&p, &msgs)); + char err[256] = {0}; + chat_msgs_request_error(&msgs, err, sizeof(err)); + TEST_ASSERT(strstr(err, cases[i].needle) != NULL); + TEST_ASSERT(strcmp(err, "invalid JSON request") != 0); + chat_msgs_free(&msgs); + free(payload); + } +} + +static void test_unsupported_image_format_names_the_format(void) { + /* GIF/WebP decode is genuinely unavailable, so the error must name the + * format rather than blaming the JSON. */ + const char gif[] = "R0lGODdhAQABAIAAAAAAAAAAACwAAAAAAQABAAACAkQBADs="; + buf uri = {0}; + buf_puts(&uri, "data:image/gif;base64,"); + buf_puts(&uri, gif); + char *payload = test_image_payload(uri.ptr); + const char *p = payload; + chat_msgs msgs = {0}; + TEST_ASSERT(!parse_messages(&p, &msgs)); + char err[256] = {0}; + chat_msgs_request_error(&msgs, err, sizeof(err)); + TEST_ASSERT(strstr(err, "GIF") != NULL); + TEST_ASSERT(strstr(err, "JPEG or PNG") != NULL); + chat_msgs_free(&msgs); + free(payload); + buf_free(&uri); +} + +static void test_stale_history_image_does_not_reject_request(void) { + /* A client replays the whole transcript on every turn, so an image the + * server cannot decode must not reject every later request: once it is no + * longer the newest message it becomes a text note. Without this, one + * WebP in the history wedges the conversation permanently. */ + const char *json = + "[{\"role\":\"user\",\"content\":[{\"type\":\"text\"," + "\"text\":\"first\"},{\"type\":\"image_url\",\"image_url\":" + "{\"url\":\"data:image/webp;base64,UklGRhIAAABXRUJQVlA4TAUAAAAvAAAAAA==\"}}]}," + "{\"role\":\"assistant\",\"content\":\"ok\"}," + "{\"role\":\"user\",\"content\":\"second\"}]"; + const char *p = json; + chat_msgs msgs = {0}; + TEST_ASSERT(parse_messages(&p, &msgs)); + TEST_ASSERT(msgs.len == 3); + /* The undecodable image is replaced in place, so the model is told it was + * there rather than answering "first" as if no image had been sent. */ + TEST_ASSERT(strstr(msgs.v[0].content, "image omitted") != NULL); + TEST_ASSERT(strstr(msgs.v[0].content, "WebP") != NULL); + TEST_ASSERT(msgs.v[0].images.len == 0); + TEST_ASSERT(!strcmp(msgs.v[2].content, "second")); + TEST_ASSERT(!msgs.image_error[0]); + chat_msgs_free(&msgs); + + /* The same image in the turn being answered is still a hard error: the + * caller just attached it and has to be told it did not arrive. */ + char *newest = test_image_payload( + "data:image/webp;base64,UklGRhIAAABXRUJQVlA4TAUAAAAvAAAAAA=="); + const char *q = newest; + chat_msgs bad = {0}; + TEST_ASSERT(!parse_messages(&q, &bad)); + char err2[256] = {0}; + chat_msgs_request_error(&bad, err2, sizeof(err2)); + TEST_ASSERT(strstr(err2, "WebP") != NULL); + chat_msgs_free(&bad); + free(newest); +} + +static void test_dropped_image_leaves_no_marker_behind(void) { + /* Images are referenced from the text by a nonce marker that rendering + * pairs with the decoded image. Dropping an image on a non-user message + * has to take its marker with it, or the marker survives into the prompt + * as literal control-character junk. */ + buf json = {0}; + buf_puts(&json, + "[{\"role\":\"assistant\",\"content\":[{\"type\":\"image_url\"," + "\"image_url\":{\"url\":\"data:image/png;base64,"); + buf_puts(&json, test_inline_png_base64); + buf_puts(&json, "\"}}]},{\"role\":\"user\",\"content\":\"hi\"}]"); + const char *p = json.ptr; + chat_msgs msgs = {0}; + TEST_ASSERT(parse_messages(&p, &msgs)); + TEST_ASSERT(msgs.len == 2); + TEST_ASSERT(msgs.v[0].images.len == 0); + TEST_ASSERT(strstr(msgs.v[0].content, "DS4_IMAGE") == NULL); + TEST_ASSERT(strchr(msgs.v[0].content, '\036') == NULL); + TEST_ASSERT(strstr(msgs.v[0].content, "image omitted") != NULL); + TEST_ASSERT(strstr(msgs.v[0].content, "user") != NULL); + chat_msgs_free(&msgs); + buf_free(&json); +} + static void test_http_image_paths_and_urls_are_rejected(void) { const char *cases[] = { "[{\"role\":\"user\",\"content\":[{\"type\":\"image_url\"," @@ -19488,6 +20004,12 @@ static void ds4_server_unit_tests_run(void) { test_thinking_canonical_with_tools_preserves_reasoning(); test_thinking_canonical_non_thinking_mode_noop(); test_openai_inline_image_content(); + test_openai_image_accepts_client_data_uri_variants(); + test_openai_image_block_without_type_is_not_dropped(); + test_rejected_image_reports_actionable_error(); + test_unsupported_image_format_names_the_format(); + test_stale_history_image_does_not_reject_request(); + test_dropped_image_leaves_no_marker_behind(); test_http_image_paths_and_urls_are_rejected(); test_anthropic_inline_image_content(); test_responses_inline_image_content(); @@ -19502,6 +20024,7 @@ static void ds4_server_unit_tests_run(void) { test_json_int_handles_non_finite_values(); test_tool_history_validation_handles_large_replays(); test_model_metadata_clamps_completion_to_context(); + test_model_metadata_advertises_image_input_with_vision(); test_live_prefix_rewind_target(); test_client_socket_nonblocking_flag(); test_client_disconnect_probe();