feat: Validate UTF-8 at NANOARROW_VALIDATION_LEVEL_FULL

src/nanoarrow/common/array.c

@@ -2123,6 +2123,155 @@ static int ArrowAssertInt8In(struct ArrowBufferView view, const int8_t* values,
   return NANOARROW_OK;
 }
 
+// Returns the length of the longest prefix of data that is valid UTF-8 (RFC 3629), so
+// data is valid UTF-8 if and only if the result is size_bytes. Overlong encodings,
+// surrogates (U+D800 to U+DFFF), code points above U+10FFFF, stray continuation bytes
+// and truncated sequences are all invalid.
+static int64_t ArrowUtf8ValidPrefix(const uint8_t* data, int64_t size_bytes) {
+  int64_t i = 0;
+  while (i < size_bytes) {
+    const uint8_t lead = data[i];
+
+    if (lead < 0x80) {
+      // After an ASCII byte, skip any further ASCII eight bytes at a time
+      i++;
+      while (size_bytes - i >= 8) {
+        uint64_t word;
+        memcpy(&word, data + i, sizeof(word));
+        if ((word & 0x8080808080808080ULL) != 0) {
+          break;
+        }
+        i += 8;
+      }
+      continue;
+    }
+
+    if (lead < 0xE0) {
+      // Two bytes (U+0080 to U+07FF); 0xC0 and 0xC1 would be overlong
+      if (lead < 0xC2 || size_bytes - i < 2 || (data[i + 1] & 0xC0) != 0x80) {
+        return i;
+      }
+      i += 2;
+      continue;
+    }
+
+    if (lead < 0xF0) {
+      // Three bytes (U+0800 to U+FFFF), excluding overlong encodings (0xE0 followed
+      // by less than 0xA0) and surrogates (0xED followed by 0xA0 or more)
+      if (size_bytes - i < 3) {
+        return i;
+      }
+      const uint8_t second = data[i + 1];
+      const uint8_t second_min = lead == 0xE0 ? 0xA0 : 0x80;
+      const uint8_t second_max = lead == 0xED ? 0x9F : 0xBF;
+      if (second < second_min || second > second_max || (data[i + 2] & 0xC0) != 0x80) {
+        return i;
+      }
+      i += 3;
+      continue;
+    }
+
+    // Four bytes (U+10000 to U+10FFFF), excluding overlong encodings (0xF0 followed by
+    // less than 0x90) and code points above U+10FFFF (0xF4 followed by 0x90 or more, or
+    // a lead byte above 0xF4)
+    if (lead > 0xF4 || size_bytes - i < 4) {
+      return i;
+    }
+    const uint8_t second = data[i + 1];
+    const uint8_t second_min = lead == 0xF0 ? 0x90 : 0x80;
+    const uint8_t second_max = lead == 0xF4 ? 0x8F : 0xBF;
+    if (second < second_min || second > second_max || (data[i + 2] & 0xC0) != 0x80 ||
+        (data[i + 3] & 0xC0) != 0x80) {
+      return i;
+    }
+    i += 4;
+  }
+
+  return size_bytes;
+}
+
+static int64_t ArrowArrayViewGetOffsetUnsafe(const struct ArrowArrayView* array_view,
+                                             int64_t i) {
+  if (array_view->layout.element_size_bits[1] == 32) {
+    return ArrowBufferViewGetInt32Unsafe(&array_view->buffer_views[1], i);
+  } else {
+    return ArrowBufferViewGetInt64Unsafe(&array_view->buffer_views[1], i);
+  }
+}
+
+static int ArrowArrayViewValidateUtf8Element(const struct ArrowArrayView* array_view,
+                                             int64_t i, const uint8_t* value,
+                                             int64_t size_bytes,
+                                             struct ArrowError* error) {
+  if (size_bytes == 0) {
+    return NANOARROW_OK;
+  }
+
+  int64_t valid_bytes = ArrowUtf8ValidPrefix(value, size_bytes);
+  if (valid_bytes != size_bytes) {
+    ArrowErrorSet(error,
+                  "[%" PRId64
+                  "] Expected valid UTF-8 %s value but found invalid "
+                  "sequence at byte %" PRId64,
+                  i, ArrowTypeString(array_view->storage_type), valid_bytes);
+    return EINVAL;
+  }
+
+  return NANOARROW_OK;
+}
+
+// Check that every non-null value of a string or large string array is valid UTF-8.
+// Like Arrow C++, values are checked one at a time (a code point split across two
+// values is invalid) and the bytes of null values are not checked.
+static int ArrowArrayViewValidateUtf8(const struct ArrowArrayView* array_view,
+                                      struct ArrowError* error) {
+  // The offsets buffer of a zero-length array may be empty
+  if (array_view->length == 0) {
+    return NANOARROW_OK;
+  }
+
+  const uint8_t* data = array_view->buffer_views[2].data.as_uint8;
+
+  // The values (including those of null slots) are the contiguous range of bytes
+  // between the first and last offsets. If that whole range is valid and every value
+  // begins on a code point boundary (i.e., not with a continuation byte), every value is
+  // valid, which avoids validating short values one at a time. Otherwise, check the
+  // non-null values one at a time to find (and report) the first invalid one.
+  {
+    int64_t first = ArrowArrayViewGetOffsetUnsafe(array_view, array_view->offset);
+    int64_t last = ArrowArrayViewGetOffsetUnsafe(array_view,
+                                                 array_view->offset + array_view->length);
+    if (first == last) {
+      return NANOARROW_OK;
+    }
+
+    int ok = ArrowUtf8ValidPrefix(data + first, last - first) == last - first;
+    for (int64_t i = 1; ok && i < array_view->length; i++) {
+      int64_t start = ArrowArrayViewGetOffsetUnsafe(array_view, array_view->offset + i);
+      ok = start == last || (data[start] & 0xC0) != 0x80;
+    }
+
+    if (ok) {
+      return NANOARROW_OK;
+    }
+  }
+
+  for (int64_t i = 0; i < array_view->length; i++) {
+    if (ArrowArrayViewIsNull(array_view, i)) {
+      continue;
+    }
+
+    int64_t start = ArrowArrayViewGetOffsetUnsafe(array_view, array_view->offset + i);
+    int64_t end = ArrowArrayViewGetOffsetUnsafe(array_view, array_view->offset + i + 1);
+    if (end > start) {
+      NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateUtf8Element(
+          array_view, i, data + start, end - start, error));
+    }
+  }
+
+  return NANOARROW_OK;
+}
+
 static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view,
                                       struct ArrowError* error) {
   for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {
@@ -2152,6 +2301,15 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view,
     }
   }
 
+  switch (array_view->storage_type) {
+    case NANOARROW_TYPE_STRING:
+    case NANOARROW_TYPE_LARGE_STRING:
+      NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateUtf8(array_view, error));
+      break;
+    default:
+      break;
+  }
+
   if (array_view->storage_type == NANOARROW_TYPE_DENSE_UNION ||
       array_view->storage_type == NANOARROW_TYPE_SPARSE_UNION) {
     struct ArrowBufferView sliced_type_ids;

src/nanoarrow/common/array_test.cc

@@ -20,7 +20,10 @@
 #include <cmath>
 #include <cstdint>
 #include <cstring>
+#include <string>
 #include <thread>
+#include <utility>
+#include <vector>
 
 #if defined(NANOARROW_BUILD_TESTS_WITH_ARROW)
 #include <arrow/array.h>
@@ -3015,6 +3018,167 @@ TEST(ArrayTest, ArrayViewTestValidateUnalignedOffsets) {
   }
 }
 
+// A value of a string or binary test array, or a null (kNullValue)
+struct TestStringValue {
+  TestStringValue(const char* value) : is_null(false), value(value) {}
+  TestStringValue(std::string value) : is_null(false), value(std::move(value)) {}
+  TestStringValue() : is_null(true) {}
+
+  bool is_null;
+  std::string value;
+};
+
+static const TestStringValue kNullValue;
+
+// Build an array of type from values, slice it, and validate it at level. On failure,
+// *message holds the error message.
+static ArrowErrorCode ValidateStringValues(enum ArrowType type,
+                                           const std::vector<TestStringValue>& values,
+                                           enum ArrowValidationLevel level,
+                                           std::string* message, int64_t offset = 0,
+                                           int64_t length = -1) {
+  nanoarrow::UniqueArray array;
+  nanoarrow::UniqueArrayView array_view;
+  struct ArrowError error;
+  error.message[0] = '\0';
+
+  NANOARROW_RETURN_NOT_OK(ArrowArrayInitFromType(array.get(), type));
+  NANOARROW_RETURN_NOT_OK(ArrowArrayStartAppending(array.get()));
+  for (const auto& value : values) {
+    if (value.is_null) {
+      NANOARROW_RETURN_NOT_OK(ArrowArrayAppendNull(array.get(), 1));
+    } else {
+      struct ArrowStringView sv = {value.value.data(),
+                                   static_cast<int64_t>(value.value.size())};
+      NANOARROW_RETURN_NOT_OK(ArrowArrayAppendString(array.get(), sv));
+    }
+  }
+  NANOARROW_RETURN_NOT_OK(
+      ArrowArrayFinishBuilding(array.get(), NANOARROW_VALIDATION_LEVEL_NONE, &error));
+
+  array->offset = offset;
+  array->length = length == -1 ? array->length - offset : length;
+
+  ArrowArrayViewInitFromType(array_view.get(), type);
+  NANOARROW_RETURN_NOT_OK(ArrowArrayViewSetArray(array_view.get(), array.get(), &error));
+  ArrowErrorCode result = ArrowArrayViewValidate(array_view.get(), level, &error);
+  *message = error.message;
+  return result;
+}
+
+TEST(ArrayTest, ArrayViewTestValidateUtf8) {
+  std::string message;
+  const std::string long_ascii(40, 'a');
+
+  for (const auto type : {NANOARROW_TYPE_STRING, NANOARROW_TYPE_LARGE_STRING}) {
+    SCOPED_TRACE(ArrowTypeString(type));
+    const std::string type_name = ArrowTypeString(type);
+    auto invalid_message = [&](int64_t i, int64_t byte) {
+      return "[" + std::to_string(i) + "] Expected valid UTF-8 " + type_name +
+             " value but found invalid sequence at byte " + std::to_string(byte);
+    };
+
+    // Valid ASCII and two-, three- and four-byte sequences
+    EXPECT_EQ(
+        ValidateStringValues(type,
+                             {"", "abc", "\xc3\xa9t\xc3\xa9", "\xe2\x82\xac",
+                              "\xf0\x9f\x98\x80", "\xf4\x8f\xbf\xbf", "\xef\xbf\xbf",
+                              long_ascii + "\xc3\xa9" + long_ascii, kNullValue},
+                             NANOARROW_VALIDATION_LEVEL_FULL, &message),
+        NANOARROW_OK)
+        << message;
+
+    // Each kind of invalid sequence
+    const std::vector<std::pair<std::string, int64_t>> invalid = {
+        {"\x80", 0},                         // Stray continuation byte
+        {"ab\xc3", 2},                       // Truncated two-byte sequence
+        {"\xe2\x82", 0},                     // Truncated three-byte sequence
+        {"\xf0\x9f\x98", 0},                 // Truncated four-byte sequence
+        {"\xc3\x28", 0},                     // Invalid continuation byte
+        {"\xc0\x80", 0},                     // Overlong two-byte encoding
+        {"\xc1\xbf", 0},                     // Overlong two-byte encoding
+        {"\xe0\x80\x80", 0},                 // Overlong three-byte encoding
+        {"\xf0\x80\x80\x80", 0},             // Overlong four-byte encoding
+        {"\xed\xa0\x80", 0},                 // Surrogate (U+D800)
+        {"\xed\xbf\xbf", 0},                 // Surrogate (U+DFFF)
+        {"\xf4\x90\x80\x80", 0},             // Above U+10FFFF
+        {"\xf5\x80\x80\x80", 0},             // Above U+10FFFF
+        {"\xff", 0},                         // Never valid
+        {"abcdefg\xffhijklmno", 7},          // Within an eight-byte word
+        {long_ascii + "\xff", 40},           // After the ASCII fast path
+        {long_ascii + "\xc3\xa9\xa9", 42}};  // After a valid multi-byte sequence
+    for (const auto& item : invalid) {
+      SCOPED_TRACE(item.second);
+      EXPECT_EQ(ValidateStringValues(type, {"abc", item.first},
+                                     NANOARROW_VALIDATION_LEVEL_FULL, &message),
+                EINVAL);
+      EXPECT_EQ(message, invalid_message(1, item.second));
+
+      // ...and with a null in the array
+      EXPECT_EQ(ValidateStringValues(type, {kNullValue, item.first},
+                                     NANOARROW_VALIDATION_LEVEL_FULL, &message),
+                EINVAL);
+      EXPECT_EQ(message, invalid_message(1, item.second));
+
+      // Invalid UTF-8 is still valid below NANOARROW_VALIDATION_LEVEL_FULL
+      EXPECT_EQ(ValidateStringValues(type, {"abc", item.first},
+                                     NANOARROW_VALIDATION_LEVEL_DEFAULT, &message),
+                NANOARROW_OK);
+    }
+
+    // Values are checked one at a time: a code point split across two values is
+    // invalid even though their concatenation is valid
+    EXPECT_EQ(ValidateStringValues(type, {"a\xc3", "\xa9"},
+                                   NANOARROW_VALIDATION_LEVEL_FULL, &message),
+              EINVAL);
+    EXPECT_EQ(message, invalid_message(0, 1));
+
+    // The bytes of a null value are not checked
+    EXPECT_EQ(ValidateStringValues(type, {"abc", "\xff", long_ascii + "\xff", "def"},
+                                   NANOARROW_VALIDATION_LEVEL_FULL, &message),
+              EINVAL);
+    nanoarrow::UniqueArray with_nulls;
+    ASSERT_EQ(ArrowArrayInitFromType(with_nulls.get(), type), NANOARROW_OK);
+    ASSERT_EQ(ArrowArrayStartAppending(with_nulls.get()), NANOARROW_OK);
+    ASSERT_EQ(ArrowArrayAppendString(with_nulls.get(), "abc"_asv), NANOARROW_OK);
+    ASSERT_EQ(ArrowArrayAppendString(with_nulls.get(), "\xff"_asv), NANOARROW_OK);
+    ASSERT_EQ(ArrowArrayAppendString(with_nulls.get(),
+                                     {long_ascii.data(), (int64_t)long_ascii.size()}),
+              NANOARROW_OK);
+    ASSERT_EQ(ArrowArrayFinishBuildingDefault(with_nulls.get(), nullptr), NANOARROW_OK);
+    // Invalidate the third value and mark the second and third values null
+    ASSERT_EQ(ArrowBitmapReserve(ArrowArrayValidityBitmap(with_nulls.get()), 3),
+              NANOARROW_OK);
+    ArrowBitmapAppendUnsafe(ArrowArrayValidityBitmap(with_nulls.get()), 1, 1);
+    ArrowBitmapAppendUnsafe(ArrowArrayValidityBitmap(with_nulls.get()), 0, 2);
+    with_nulls->null_count = 2;
+    ArrowArrayBuffer(with_nulls.get(), 2)->data[4] = 0xff;
+    struct ArrowError error;
+    EXPECT_EQ(ArrowArrayFinishBuilding(with_nulls.get(), NANOARROW_VALIDATION_LEVEL_FULL,
+                                       &error),
+              NANOARROW_OK)
+        << error.message;
+
+    // Only the values within a slice are checked, and indices are relative to it
+    EXPECT_EQ(ValidateStringValues(type, {"\xff", "abc", "def", "\xff"},
+                                   NANOARROW_VALIDATION_LEVEL_FULL, &message, 1, 2),
+              NANOARROW_OK)
+        << message;
+    EXPECT_EQ(ValidateStringValues(type, {"\xff", "abc", "\xff", "def"},
+                                   NANOARROW_VALIDATION_LEVEL_FULL, &message, 1, 2),
+              EINVAL);
+    EXPECT_EQ(message, invalid_message(1, 0));
+  }
+
+  // Binary values are not checked
+  for (const auto type : {NANOARROW_TYPE_BINARY, NANOARROW_TYPE_LARGE_BINARY}) {
+    EXPECT_EQ(ValidateStringValues(type, {"\xff", long_ascii + "\xff"},
+                                   NANOARROW_VALIDATION_LEVEL_FULL, &message),
+              NANOARROW_OK)
+        << message;
+  }
+}
+
 TEST(ArrayTest, ArrayViewTestLargeString) {
   struct ArrowArrayView array_view;
   struct ArrowError error;

src/nanoarrow/common/inline_types.h

@@ -601,6 +601,8 @@ enum ArrowValidationLevel {
 
   /// \brief Validate all buffer sizes and all buffer content. This is useful in the
   /// context of untrusted input or input that may have been corrupted in transit.
+  /// This includes checking that every non-null string and large string value is
+  /// valid UTF-8.
   NANOARROW_VALIDATION_LEVEL_FULL = 3
 };