From 3a9b83b8bad66da9873b24f7b3b3f7b4cd9ccca8 Mon Sep 17 00:00:00 2001 From: "t.hirashima" Date: Tue, 22 Sep 2026 01:46:26 +0900 Subject: [PATCH 1/2] Fix low surrogate in UTF-16 output The high byte of the low surrogate was computed with the mask 0xFF instead of 0x03, so it also picked up bits that belong to the high surrogate. For example U+1F310 was written as D83C CF10 instead of D83C DF10. This code was unreachable so far, because the emitter always escapes characters above U+FFFF as "\UXXXXXXXX". --- src/writer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/writer.c b/src/writer.c index 5d57f392..88a0c2f9 100644 --- a/src/writer.c +++ b/src/writer.c @@ -116,7 +116,7 @@ yaml_emitter_flush(yaml_emitter_t *emitter) value -= 0x10000; emitter->raw_buffer.last[high] = 0xD8 + (value >> 18); emitter->raw_buffer.last[low] = (value >> 10) & 0xFF; - emitter->raw_buffer.last[high+2] = 0xDC + ((value >> 8) & 0xFF); + emitter->raw_buffer.last[high+2] = 0xDC + ((value >> 8) & 0x03); emitter->raw_buffer.last[low+2] = value & 0xFF; emitter->raw_buffer.last += 4; From 3bcd5ff1d31cd4465ac4f8e50a561216db591a19 Mon Sep 17 00:00:00 2001 From: "t.hirashima" Date: Tue, 22 Sep 2026 01:46:26 +0900 Subject: [PATCH 2/2] Treat #x10000-#x10FFFF as printable in the emitter IS_PRINTABLE covered only the 1-3 byte ranges of c-printable. Even with yaml_emitter_set_unicode(emitter, 1), characters outside the BMP (e.g. emoji) were escaped as "\U0001F310", which also forced the scalar into double quoted style. c-printable includes [#x10000-#x10FFFF] in YAML 1.1 and 1.2. Add the 4 byte range to the macro. The upper bound is checked exactly, because yaml_check_utf8() does not reject sequences above U+10FFFF. Add tests/test-emitter.c to check the escaping and the UTF-16 output of such characters. Fixes #279 --- .gitignore | 1 + src/yaml_private.h | 8 +- tests/CMakeLists.txt | 2 + tests/Makefile.am | 4 +- tests/test-emitter.c | 172 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 tests/test-emitter.c diff --git a/.gitignore b/.gitignore index 637273f1..feeb63b1 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ stamp-h1 /tests/example-reformatter /tests/example-reformatter-alt /tests/run-test-suite +/tests/test-emitter /tests/test-nesting /tests/test-reader /tests/test-version diff --git a/src/yaml_private.h b/src/yaml_private.h index b3351c41..b95805d1 100644 --- a/src/yaml_private.h +++ b/src/yaml_private.h @@ -274,7 +274,13 @@ yaml_string_join( && (string).pointer[offset+2] == 0xBF) \ && !((string).pointer[offset+1] == 0xBF \ && ((string).pointer[offset+2] == 0xBE \ - || (string).pointer[offset+2] == 0xBF)))) + || (string).pointer[offset+2] == 0xBF))) \ + || ((string).pointer[offset] == 0xF0 /* #x10000 <= . <= #x10FFFF */ \ + && (string).pointer[offset+1] >= 0x90) \ + || ((string).pointer[offset] > 0xF0 \ + && (string).pointer[offset] < 0xF4) \ + || ((string).pointer[offset] == 0xF4 \ + && (string).pointer[offset+1] < 0x90)) #define IS_PRINTABLE(string) IS_PRINTABLE_AT((string),0) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 17a9c671..96a9765a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -16,6 +16,7 @@ foreach(name IN ITEMS run-parser run-parser-test-suite run-scanner + test-emitter test-nesting test-reader test-version @@ -26,4 +27,5 @@ endforeach() add_test(NAME version COMMAND test-version) add_test(NAME reader COMMAND test-reader) add_test(NAME nesting COMMAND test-nesting) +add_test(NAME emitter COMMAND test-emitter) diff --git a/tests/Makefile.am b/tests/Makefile.am index 8b8a456f..e104163b 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,8 +1,8 @@ AM_CPPFLAGS = -I$(top_srcdir)/include -Wall #AM_CFLAGS = -Wno-pointer-sign LDADD = $(top_builddir)/src/libyaml.la -TESTS = test-version test-reader test-nesting -check_PROGRAMS = test-version test-reader test-nesting +TESTS = test-version test-reader test-nesting test-emitter +check_PROGRAMS = test-version test-reader test-nesting test-emitter noinst_PROGRAMS = run-scanner run-parser run-loader run-emitter run-dumper \ example-reformatter example-reformatter-alt \ example-deconstructor example-deconstructor-alt \ diff --git a/tests/test-emitter.c b/tests/test-emitter.c new file mode 100644 index 00000000..a7d2cbdd --- /dev/null +++ b/tests/test-emitter.c @@ -0,0 +1,172 @@ +#include + +#include +#include +#include + +#ifdef NDEBUG +#undef NDEBUG +#endif +#include + +/* + * Test that the emitter writes characters outside the Basic Multilingual + * Plane (#x10000-#x10FFFF, 4 bytes in UTF-8) unescaped if unescaped unicode + * output is allowed, and that they are encoded correctly in UTF-16. + */ + +#define BUFFER_SIZE 256 + +typedef struct { + unsigned char data[BUFFER_SIZE]; + size_t length; +} output_t; + +typedef struct { + const char *title; + yaml_encoding_t encoding; + int unicode; + yaml_scalar_style_t style; + const char *value; + const char *expected; + size_t expected_length; +} test_case; + +#define UTF8 YAML_UTF8_ENCODING +#define UTF16LE YAML_UTF16LE_ENCODING +#define UTF16BE YAML_UTF16BE_ENCODING +#define PLAIN YAML_PLAIN_SCALAR_STYLE +#define DQUOTED YAML_DOUBLE_QUOTED_SCALAR_STYLE + +test_case unicode_scalars[] = { + /* {"title", encoding, unicode, style, "value", "expected", length}, */ + + {"u-2600 is written unescaped", UTF8, 1, PLAIN, + "\xe2\x98\x80", "\xe2\x98\x80\n", 4}, + {"u-FFFE is escaped", UTF8, 1, PLAIN, + "\xef\xbf\xbe", "\"\\uFFFE\"\n", 9}, + {"u-10000 is written unescaped", UTF8, 1, PLAIN, + "\xf0\x90\x80\x80", "\xf0\x90\x80\x80\n", 5}, + {"u-1F310 is written unescaped", UTF8, 1, PLAIN, + "\xf0\x9f\x8c\x90", "\xf0\x9f\x8c\x90\n", 5}, + {"u-40000 is written unescaped", UTF8, 1, PLAIN, + "\xf1\x80\x80\x80", "\xf1\x80\x80\x80\n", 5}, + {"u-10FFFF is written unescaped", UTF8, 1, PLAIN, + "\xf4\x8f\xbf\xbf", "\xf4\x8f\xbf\xbf\n", 5}, + /* Not a Unicode character, but yaml_scalar_event_initialize accepts it; + * it must not be written unescaped. */ + {"u-110000 is escaped", UTF8, 1, PLAIN, + "\xf4\x90\x80\x80", "\"\\U00110000\"\n", 13}, + {"u-1F310 is written unescaped in a double quoted scalar", UTF8, 1, DQUOTED, + "\xf0\x9f\x8c\x90", "\"\xf0\x9f\x8c\x90\"\n", 7}, + {"u-1F310 is escaped if unicode is not allowed", UTF8, 0, PLAIN, + "\xf0\x9f\x8c\x90", "\"\\U0001F310\"\n", 13}, + + {"u-1F310 in utf-16-le", UTF16LE, 1, PLAIN, + "\xf0\x9f\x8c\x90", "\xff\xfe\x3c\xd8\x10\xdf\x0a\x00", 8}, + {"u-1F310 in utf-16-be", UTF16BE, 1, PLAIN, + "\xf0\x9f\x8c\x90", "\xfe\xff\xd8\x3c\xdf\x10\x00\x0a", 8}, + {"u-10FFFF in utf-16-le", UTF16LE, 1, PLAIN, + "\xf4\x8f\xbf\xbf", "\xff\xfe\xff\xdb\xff\xdf\x0a\x00", 8}, + + {NULL, UTF8, 0, PLAIN, NULL, NULL, 0} +}; + +static int +write_handler(void *data, unsigned char *buffer, size_t size) +{ + output_t *output = (output_t *)data; + + if (output->length + size > BUFFER_SIZE) + return 0; + memcpy(output->data + output->length, buffer, size); + output->length += size; + return 1; +} + +static int +emit_scalar(test_case *test, output_t *output) +{ + yaml_emitter_t emitter; + yaml_event_t event; + + output->length = 0; + + assert(yaml_emitter_initialize(&emitter)); + yaml_emitter_set_output(&emitter, write_handler, output); + yaml_emitter_set_unicode(&emitter, test->unicode); + + if (!yaml_stream_start_event_initialize(&event, test->encoding) + || !yaml_emitter_emit(&emitter, &event)) + goto error; + if (!yaml_document_start_event_initialize(&event, NULL, NULL, NULL, 1) + || !yaml_emitter_emit(&emitter, &event)) + goto error; + if (!yaml_scalar_event_initialize(&event, NULL, NULL, + (yaml_char_t *)test->value, (int)strlen(test->value), + 1, 1, test->style) + || !yaml_emitter_emit(&emitter, &event)) + goto error; + if (!yaml_document_end_event_initialize(&event, 1) + || !yaml_emitter_emit(&emitter, &event)) + goto error; + if (!yaml_stream_end_event_initialize(&event) + || !yaml_emitter_emit(&emitter, &event)) + goto error; + + yaml_emitter_delete(&emitter); + return 1; + +error: + yaml_emitter_delete(&emitter); + return 0; +} + +static void +print_bytes(const unsigned char *data, size_t length) +{ + size_t k; + + for (k = 0; k < length; k ++) { + printf("%02X ", data[k]); + } +} + +int check_unicode_scalars(void) +{ + output_t output; + int failed = 0; + int k; + + printf("checking unicode scalars...\n"); + for (k = 0; unicode_scalars[k].title; k ++) { + test_case *test = unicode_scalars+k; + + printf("\t%s: ", test->title); + if (!emit_scalar(test, &output)) { + printf("- (emitter error)\n"); + failed ++; + } + else if (output.length != test->expected_length + || memcmp(output.data, test->expected, output.length) != 0) { + printf("- (got "); + print_bytes(output.data, output.length); + printf("while expected "); + print_bytes((const unsigned char *)test->expected, + test->expected_length); + printf(")\n"); + failed ++; + } + else { + printf("+\n"); + } + } + printf("checking unicode scalars: %d fail(s)\n", failed); + return failed; +} + +int +main(void) +{ + return check_unicode_scalars(); +}