From 12278b3a5f2b8364ee054882848051efbb3108cf Mon Sep 17 00:00:00 2001 From: ump45nose <52391318+ump45nose@users.noreply.github.com> Date: Thu, 22 Oct 2026 05:00:23 +0800 Subject: [PATCH 1/2] fix: Avoid implicit 64-to-32-bit conversions on 32-bit platforms Building with -Wconversion on 32-bit targets (e.g. armv7 Android in the vcpkg CI) fails because several int64_t values are passed to functions that take size_t (memcpy, memset, snprintf, malloc, fread and flatcc builder helpers). Cast these explicitly. Closes #797. --- src/nanoarrow/common/array.c | 7 +++-- src/nanoarrow/common/inline_array.h | 2 +- src/nanoarrow/common/inline_buffer.h | 4 +-- src/nanoarrow/common/inline_types.h | 2 +- src/nanoarrow/common/schema.c | 21 +++++++------- src/nanoarrow/common/utils.c | 13 +++++---- src/nanoarrow/device/device.c | 2 +- src/nanoarrow/ipc/decoder.c | 11 ++++---- src/nanoarrow/ipc/encoder.c | 42 ++++++++++++++++------------ src/nanoarrow/ipc/reader.c | 8 ++++-- src/nanoarrow/ipc/writer.c | 3 +- 11 files changed, 64 insertions(+), 51 deletions(-) diff --git a/src/nanoarrow/common/array.c b/src/nanoarrow/common/array.c index e4538955c..f9773c8f3 100644 --- a/src/nanoarrow/common/array.c +++ b/src/nanoarrow/common/array.c @@ -854,7 +854,7 @@ ArrowErrorCode ArrowArrayAllocateChildren(struct ArrowArray* array, int64_t n_ch return ENOMEM; } - memset(array->children, 0, n_children * sizeof(struct ArrowArray*)); + memset(array->children, 0, (size_t)n_children * sizeof(struct ArrowArray*)); for (int64_t i = 0; i < n_children; i++) { array->children[i] = (struct ArrowArray*)ArrowMalloc(sizeof(struct ArrowArray)); @@ -2363,8 +2363,9 @@ static void ArrowArrayViewCompareBuffer(const struct ArrowArrayView* actual, SET_NOT_EQUAL_AND_RETURN_IF( actual->buffer_views[i].size_bytes != expected->buffer_views[i].size_bytes, state); - int64_t buffer_size = actual->buffer_views[i].size_bytes; - if (buffer_size > 0) { + int64_t size_bytes = actual->buffer_views[i].size_bytes; + if (size_bytes > 0) { + size_t buffer_size = (size_t)size_bytes; SET_NOT_EQUAL_AND_RETURN_IF( memcmp(actual->buffer_views[i].data.data, expected->buffer_views[i].data.data, buffer_size) != 0, diff --git a/src/nanoarrow/common/inline_array.h b/src/nanoarrow/common/inline_array.h index eaec776e0..5f75326f2 100644 --- a/src/nanoarrow/common/inline_array.h +++ b/src/nanoarrow/common/inline_array.h @@ -594,7 +594,7 @@ static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array, if (value.size_bytes <= NANOARROW_BINARY_VIEW_INLINE_SIZE) { if (value.size_bytes > 0) { - memcpy(bvt.inlined.data, value.data.as_char, value.size_bytes); + memcpy(bvt.inlined.data, value.data.as_char, (size_t)value.size_bytes); } memset(bvt.inlined.data + bvt.inlined.size, 0, NANOARROW_BINARY_VIEW_INLINE_SIZE - bvt.inlined.size); diff --git a/src/nanoarrow/common/inline_buffer.h b/src/nanoarrow/common/inline_buffer.h index bd74afb48..0694264ec 100644 --- a/src/nanoarrow/common/inline_buffer.h +++ b/src/nanoarrow/common/inline_buffer.h @@ -247,7 +247,7 @@ static inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const voi int64_t size_bytes) { if (size_bytes > 0) { NANOARROW_DCHECK(buffer->data != NULL); - memcpy(buffer->data + buffer->size_bytes, data, size_bytes); + memcpy(buffer->data + buffer->size_bytes, data, (size_t)size_bytes); buffer->size_bytes += size_bytes; } } @@ -329,7 +329,7 @@ static inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer, NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes)); NANOARROW_DCHECK(buffer->data != NULL); // To help clang-tidy - memset(buffer->data + buffer->size_bytes, value, size_bytes); + memset(buffer->data + buffer->size_bytes, value, (size_t)size_bytes); buffer->size_bytes += size_bytes; return NANOARROW_OK; diff --git a/src/nanoarrow/common/inline_types.h b/src/nanoarrow/common/inline_types.h index 1c1cd0bbc..9b19d918b 100644 --- a/src/nanoarrow/common/inline_types.h +++ b/src/nanoarrow/common/inline_types.h @@ -265,7 +265,7 @@ static inline void ArrowErrorSetString(struct ArrowError* error, const char* src memcpy(error->message, src, sizeof(error->message) - 1); error->message[sizeof(error->message) - 1] = '\0'; } else { - memcpy(error->message, src, src_len); + memcpy(error->message, src, (size_t)src_len); error->message[src_len] = '\0'; } } diff --git a/src/nanoarrow/common/schema.c b/src/nanoarrow/common/schema.c index 76d79b01c..997602a45 100644 --- a/src/nanoarrow/common/schema.c +++ b/src/nanoarrow/common/schema.c @@ -434,18 +434,18 @@ ArrowErrorCode ArrowSchemaSetTypeUnion(struct ArrowSchema* schema, enum ArrowTyp // Max valid size would be +ud:0,1,...126 = 401 characters + null terminator char format_out[512]; int64_t format_out_size = 512; - memset(format_out, 0, format_out_size); + memset(format_out, 0, (size_t)format_out_size); int n_chars; char* format_cursor = format_out; switch (type) { case NANOARROW_TYPE_SPARSE_UNION: - n_chars = snprintf(format_cursor, format_out_size, "+us:"); + n_chars = snprintf(format_cursor, (size_t)format_out_size, "+us:"); format_cursor += n_chars; format_out_size -= n_chars; break; case NANOARROW_TYPE_DENSE_UNION: - n_chars = snprintf(format_cursor, format_out_size, "+ud:"); + n_chars = snprintf(format_cursor, (size_t)format_out_size, "+ud:"); format_cursor += n_chars; format_out_size -= n_chars; break; @@ -460,12 +460,12 @@ ArrowErrorCode ArrowSchemaSetTypeUnion(struct ArrowSchema* schema, enum ArrowTyp } if (n_children > 0) { - n_chars = snprintf(format_cursor, format_out_size, "0"); + n_chars = snprintf(format_cursor, (size_t)format_out_size, "0"); format_cursor += n_chars; format_out_size -= n_chars; for (int64_t i = 1; i < n_children; i++) { - n_chars = snprintf(format_cursor, format_out_size, ",%" PRId64, i); + n_chars = snprintf(format_cursor, (size_t)format_out_size, ",%" PRId64, i); format_cursor += n_chars; format_out_size -= n_chars; } @@ -533,7 +533,7 @@ ArrowErrorCode ArrowSchemaSetMetadata(struct ArrowSchema* schema, const char* me } if (metadata != NULL) { - size_t metadata_size = ArrowMetadataSizeOf(metadata); + size_t metadata_size = (size_t)ArrowMetadataSizeOf(metadata); schema->metadata = (const char*)ArrowMalloc(metadata_size); if (schema->metadata == NULL) { return ENOMEM; @@ -563,7 +563,7 @@ ArrowErrorCode ArrowSchemaAllocateChildren(struct ArrowSchema* schema, schema->n_children = n_children; - memset(schema->children, 0, n_children * sizeof(struct ArrowSchema*)); + memset(schema->children, 0, (size_t)n_children * sizeof(struct ArrowSchema*)); for (int64_t i = 0; i < n_children; i++) { schema->children[i] = (struct ArrowSchema*)ArrowMalloc(sizeof(struct ArrowSchema)); @@ -1616,8 +1616,9 @@ static ArrowErrorCode ArrowMetadataGetValueInternal(const char* metadata, while (ArrowMetadataReaderRead(&reader, &existing_key, &existing_value) == NANOARROW_OK) { - int key_equal = key->size_bytes == existing_key.size_bytes && - strncmp(key->data, existing_key.data, existing_key.size_bytes) == 0; + int key_equal = + key->size_bytes == existing_key.size_bytes && + strncmp(key->data, existing_key.data, (size_t)existing_key.size_bytes) == 0; if (key_equal) { value_out->data = existing_value.data; value_out->size_bytes = existing_value.size_bytes; @@ -1721,7 +1722,7 @@ static ArrowErrorCode ArrowMetadataBuilderSetInternal(struct ArrowBuffer* buffer if (key->size_bytes == existing_key.size_bytes && strncmp((const char*)key->data, (const char*)existing_key.data, - existing_key.size_bytes) == 0) { + (size_t)existing_key.size_bytes) == 0) { result = ArrowMetadataBuilderAppendInternal(&new_buffer, key, value); value = NULL; } else { diff --git a/src/nanoarrow/common/utils.c b/src/nanoarrow/common/utils.c index c86d00b38..a2e2ca1ad 100644 --- a/src/nanoarrow/common/utils.c +++ b/src/nanoarrow/common/utils.c @@ -237,9 +237,9 @@ void ArrowLayoutInit(struct ArrowLayout* layout, enum ArrowType storage_type) { } } -void* ArrowMalloc(int64_t size) { return malloc(size); } +void* ArrowMalloc(int64_t size) { return malloc((size_t)size); } -void* ArrowRealloc(void* ptr, int64_t size) { return realloc(ptr, size); } +void* ArrowRealloc(void* ptr, int64_t size) { return realloc(ptr, (size_t)size); } void ArrowFree(void* ptr) { free(ptr); } @@ -542,7 +542,7 @@ static void ShiftAndAdd(struct ArrowStringView value, uint32_t* out, int64_t out const uint64_t multiple = kUInt32PowersOfTen[group_size]; - memcpy(chunk_string, value.data + posn, group_size); + memcpy(chunk_string, value.data + posn, (size_t)group_size); chunk_string[group_size] = '\0'; uint32_t chunk = (uint32_t)strtoll(chunk_string, NULL, 10); @@ -627,8 +627,9 @@ ArrowErrorCode ArrowDecimalAppendDigitsToBuffer(const struct ArrowDecimal* decim int32_t value; memcpy(&value, decimal->words, sizeof(int32_t)); NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, 16)); - int n_chars = snprintf((char*)buffer->data + buffer->size_bytes, - (buffer->capacity_bytes - buffer->size_bytes), "%d", value); + int n_chars = + snprintf((char*)buffer->data + buffer->size_bytes, + (size_t)(buffer->capacity_bytes - buffer->size_bytes), "%d", value); if (n_chars <= 0) { return EINVAL; } @@ -783,7 +784,7 @@ ArrowErrorCode ArrowDecimalAppendStringToBuffer(const struct ArrowDecimal* decim ArrowBufferResize(buffer, buffer->size_bytes + num_zeros_after_decimal + 2, 0)); uint8_t* digits_start = buffer->data + is_negative; - memmove(digits_start + num_zeros_after_decimal + 2, digits_start, num_digits); + memmove(digits_start + num_zeros_after_decimal + 2, digits_start, (size_t)num_digits); *digits_start++ = '0'; *digits_start++ = '.'; for (int i = 0; i < num_zeros_after_decimal; i++) { diff --git a/src/nanoarrow/device/device.c b/src/nanoarrow/device/device.c index a7aea2a27..88a27ba7f 100644 --- a/src/nanoarrow/device/device.c +++ b/src/nanoarrow/device/device.c @@ -89,7 +89,7 @@ static ArrowErrorCode ArrowDeviceCpuBufferCopy(struct ArrowDevice* device_src, return EINVAL; } - memcpy((uint8_t*)dst.data.as_uint8, src.data.as_uint8, dst.size_bytes); + memcpy((uint8_t*)dst.data.as_uint8, src.data.as_uint8, (size_t)dst.size_bytes); return NANOARROW_OK; } diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 80e53ec8d..15d769650 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -721,7 +721,7 @@ static ArrowErrorCode ArrowIpcDecoderVisitMetadata( int64_t n_pairs = ns(KeyValue_vec_len(kv_vec)); for (int64_t i = 0; i < n_pairs; i++) { - ns(KeyValue_table_t) kv = ns(KeyValue_vec_at(kv_vec, i)); + ns(KeyValue_table_t) kv = ns(KeyValue_vec_at(kv_vec, (size_t)i)); flatbuffers_string_t key = ns(KeyValue_key(kv)); flatbuffers_string_t value = ns(KeyValue_value(kv)); @@ -1168,7 +1168,7 @@ static int ArrowIpcDecoderSetTypeUnion(struct ArrowSchema* schema, for (int64_t i = 1; i < n_type_ids; i++) { n_chars = snprintf(format_cursor, format_out_size, ",%" PRId32, - flatbuffers_int32_vec_at(type_ids, i)); + flatbuffers_int32_vec_at(type_ids, (size_t)i)); format_cursor += n_chars; format_out_size -= n_chars; @@ -1417,7 +1417,7 @@ static int ArrowIpcDecoderSetChildren(struct ArrowSchema* schema, ns(Field_vec_t int64_t n_fields = ns(Schema_vec_len(fields)); for (int64_t i = 0; i < n_fields; i++) { - ns(Field_table_t) field = ns(Field_vec_at(fields, i)); + ns(Field_table_t) field = ns(Field_vec_at(fields, (size_t)i)); NANOARROW_RETURN_NOT_OK( ArrowIpcDecoderSetField(schema->children[i], field, dictionaries, error)); } @@ -1449,7 +1449,7 @@ static int ArrowIpcDecoderDecodeSchemaHeader(struct ArrowIpcDecoder* decoder, decoder->feature_flags = 0; for (int64_t i = 0; i < n_features; i++) { - ns(Feature_enum_t) feature = ns(Feature_vec_at(features, i)); + ns(Feature_enum_t) feature = ns(Feature_vec_at(features, (size_t)i)); switch (feature) { case ns(Feature_COMPRESSED_BODY): decoder->feature_flags |= NANOARROW_IPC_FEATURE_COMPRESSED_BODY; @@ -2086,7 +2086,8 @@ ArrowErrorCode ArrowIpcDecoderSetSchemaWithDictionaries( ArrowErrorSet(error, "Failed to allocate decoder->fields"); return ENOMEM; } - memset(private_data->fields, 0, private_data->n_fields * sizeof(struct ArrowIpcField)); + memset(private_data->fields, 0, + (size_t)private_data->n_fields * sizeof(struct ArrowIpcField)); // Init field information and calculate starting buffer offset for each int64_t field_i = 0; diff --git a/src/nanoarrow/ipc/encoder.c b/src/nanoarrow/ipc/encoder.c index ca70b244c..02577ac71 100644 --- a/src/nanoarrow/ipc/encoder.c +++ b/src/nanoarrow/ipc/encoder.c @@ -250,7 +250,8 @@ ArrowErrorCode ArrowIpcEncoderFinalizeBuffer(struct ArrowIpcEncoder* encoder, int64_t padded_size = _ArrowRoundUpToMultipleOf8(size); NANOARROW_RETURN_NOT_OK( ArrowBufferReserve(out, sizeof(int32_t) + sizeof(int32_t) + padded_size)); - NANOARROW_ASSERT_OK(ArrowIpcEncoderWriteContinuationAndSize(out, padded_size)); + NANOARROW_ASSERT_OK( + ArrowIpcEncoderWriteContinuationAndSize(out, (size_t)padded_size)); } else { NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(out, size)); } @@ -503,9 +504,10 @@ static ArrowErrorCode ArrowIpcEncodeMetadata(flatcc_builder_t* builder, NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowMetadataReaderRead(&metadata, &key, &value), error); FLATCC_RETURN_UNLESS_0_NO_NS(push_start(builder), error); - FLATCC_RETURN_UNLESS_0(KeyValue_key_create(builder, key.data, key.size_bytes), error); - FLATCC_RETURN_UNLESS_0(KeyValue_value_create(builder, value.data, value.size_bytes), + FLATCC_RETURN_UNLESS_0(KeyValue_key_create(builder, key.data, (size_t)key.size_bytes), error); + FLATCC_RETURN_UNLESS_0( + KeyValue_value_create(builder, value.data, (size_t)value.size_bytes), error); FLATCC_RETURN_IF_NULL(push_end(builder), error); } return NANOARROW_OK; @@ -822,7 +824,7 @@ static ArrowErrorCode ArrowIpcEncoderReserveCompressedBuffers( if (private->n_compressed_buffers > 0) { memcpy(buffers, private->compressed_buffers, - private->n_compressed_buffers * sizeof(struct ArrowBuffer)); + (size_t) private->n_compressed_buffers * sizeof(struct ArrowBuffer)); ArrowFree(private->compressed_buffers); } for (int64_t i = private->n_compressed_buffers; i < n; i++) { @@ -1074,14 +1076,16 @@ static ArrowErrorCode ArrowIpcEncoderEncodeRecordBatch( ArrowIpcEncoderEncodeRecordBatchImpl(private, array_view, error)); NANOARROW_RETURN_NOT_OK(ArrowIpcEncoderEncodeBuffers(encoder, buffer_encoder, error)); - FLATCC_RETURN_UNLESS_0(RecordBatch_nodes_create( // - builder, (struct ns(FieldNode)*)private->nodes.data, - private->nodes.size_bytes / sizeof(struct ns(FieldNode))), - error); - FLATCC_RETURN_UNLESS_0(RecordBatch_buffers_create( // - builder, (struct ns(Buffer)*)private->buffers.data, - private->buffers.size_bytes / sizeof(struct ns(Buffer))), - error); + FLATCC_RETURN_UNLESS_0( + RecordBatch_nodes_create( // + builder, (struct ns(FieldNode)*)private->nodes.data, + (size_t) private->nodes.size_bytes / sizeof(struct ns(FieldNode))), + error); + FLATCC_RETURN_UNLESS_0( + RecordBatch_buffers_create( // + builder, (struct ns(Buffer)*)private->buffers.data, + (size_t) private->buffers.size_bytes / sizeof(struct ns(Buffer))), + error); FLATCC_RETURN_UNLESS_0(Message_header_RecordBatch_end(builder), error); @@ -1145,12 +1149,14 @@ static ArrowErrorCode ArrowIpcEncoderEncodeDictionaryBatch( NANOARROW_RETURN_NOT_OK(ArrowIpcEncoderEncodeBuffers(encoder, buffer_encoder, error)); FLATCC_RETURN_UNLESS_0( - RecordBatch_nodes_create(builder, (struct ns(FieldNode)*)private->nodes.data, - private->nodes.size_bytes / sizeof(struct ns(FieldNode))), + RecordBatch_nodes_create( + builder, (struct ns(FieldNode)*)private->nodes.data, + (size_t) private->nodes.size_bytes / sizeof(struct ns(FieldNode))), error); FLATCC_RETURN_UNLESS_0( - RecordBatch_buffers_create(builder, (struct ns(Buffer)*)private->buffers.data, - private->buffers.size_bytes / sizeof(struct ns(Buffer))), + RecordBatch_buffers_create( + builder, (struct ns(Buffer)*)private->buffers.data, + (size_t) private->buffers.size_bytes / sizeof(struct ns(Buffer))), error); FLATCC_RETURN_UNLESS_0(DictionaryBatch_data_end(builder), error); FLATCC_RETURN_UNLESS_0(DictionaryBatch_isDelta_add(builder, is_delta ? 1 : 0), error); @@ -1224,7 +1230,7 @@ ArrowErrorCode ArrowIpcEncoderEncodeFooter(struct ArrowIpcEncoder* encoder, FLATCC_RETURN_UNLESS_0(Footer_recordBatches_start(builder), error); struct ns(Block)* flatcc_RecordBatch_blocks = - ns(Footer_recordBatches_extend(builder, n_blocks)); + ns(Footer_recordBatches_extend(builder, (size_t)n_blocks)); FLATCC_RETURN_IF_NULL(flatcc_RecordBatch_blocks, error); for (int64_t i = 0; i < n_blocks; i++) { struct ns(Block) block = { @@ -1243,7 +1249,7 @@ ArrowErrorCode ArrowIpcEncoderEncodeFooter(struct ArrowIpcEncoder* encoder, FLATCC_RETURN_UNLESS_0(Footer_dictionaries_start(builder), error); struct ns(Block)* flatcc_dict_blocks = - ns(Footer_dictionaries_extend(builder, n_dict_blocks)); + ns(Footer_dictionaries_extend(builder, (size_t)n_dict_blocks)); FLATCC_RETURN_IF_NULL(flatcc_dict_blocks, error); for (int64_t i = 0; i < n_dict_blocks; i++) { struct ns(Block) block = { diff --git a/src/nanoarrow/ipc/reader.c b/src/nanoarrow/ipc/reader.c index 1c66cc316..8e6e5c6b3 100644 --- a/src/nanoarrow/ipc/reader.c +++ b/src/nanoarrow/ipc/reader.c @@ -66,7 +66,8 @@ static ArrowErrorCode ArrowIpcInputStreamBufferRead(struct ArrowIpcInputStream* } if (bytes_to_read > 0) { - memcpy(buf, private_data->input.data + private_data->cursor_bytes, bytes_to_read); + memcpy(buf, private_data->input.data + private_data->cursor_bytes, + (size_t)bytes_to_read); } *size_read_out = bytes_to_read; @@ -133,7 +134,8 @@ static ArrowErrorCode ArrowIpcInputStreamFileRead(struct ArrowIpcInputStream* st } // Do the read - int64_t bytes_read = (int64_t)fread(buf, 1, buf_size_bytes, private_data->file_ptr); + int64_t bytes_read = + (int64_t)fread(buf, 1, (size_t)buf_size_bytes, private_data->file_ptr); *size_read_out = bytes_read; if (bytes_read != buf_size_bytes) { @@ -293,7 +295,7 @@ static int ArrowIpcArrayStreamReaderNextHeader( NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowBufferReserve(&private_data->header, 4), &private_data->error); memmove(private_data->header.data + 4, private_data->header.data, - private_data->header.size_bytes); + (size_t)private_data->header.size_bytes); uint32_t continuation = 0xFFFFFFFFU; memcpy(private_data->header.data, &continuation, sizeof(uint32_t)); private_data->header.size_bytes += 4; diff --git a/src/nanoarrow/ipc/writer.c b/src/nanoarrow/ipc/writer.c index fca3f308a..b0116d699 100644 --- a/src/nanoarrow/ipc/writer.c +++ b/src/nanoarrow/ipc/writer.c @@ -123,7 +123,8 @@ static ArrowErrorCode ArrowIpcOutputStreamFileWrite(struct ArrowIpcOutputStream* } // Do the write - int64_t bytes_written = (int64_t)fwrite(buf, 1, buf_size_bytes, private_data->file_ptr); + int64_t bytes_written = + (int64_t)fwrite(buf, 1, (size_t)buf_size_bytes, private_data->file_ptr); *size_written_out = bytes_written; if (bytes_written != buf_size_bytes) { From 487d8a914813ae5862699e1d58eb9e6180b9b9c1 Mon Sep 17 00:00:00 2001 From: ump45nose <52391318+ump45nose@users.noreply.github.com> Date: Thu, 22 Oct 2026 23:24:36 +0800 Subject: [PATCH 2/2] Check int64-to-size_t conversions in debug builds and return errors where input-derived Add _ArrowInt64FitsInSizeT()/_ArrowInt64ToSizeT() to inline_types.h and use them for the conversions touched in this PR. ArrowMalloc()/ArrowRealloc() return NULL (freeing the pointer in the realloc case) for sizes that do not fit in size_t, the IPC encoder checks the padded message size instead of asserting, and the stream reader returns EINVAL if the header size is out of range. --- src/nanoarrow/common/array.c | 4 ++-- src/nanoarrow/common/inline_array.h | 3 ++- src/nanoarrow/common/inline_buffer.h | 4 ++-- src/nanoarrow/common/inline_types.h | 16 ++++++++++++++ src/nanoarrow/common/schema.c | 24 +++++++++++---------- src/nanoarrow/common/utils.c | 28 ++++++++++++++++++------ src/nanoarrow/device/device.c | 3 ++- src/nanoarrow/ipc/decoder.c | 10 ++++----- src/nanoarrow/ipc/encoder.c | 32 +++++++++++++++------------- src/nanoarrow/ipc/reader.c | 10 +++++++-- src/nanoarrow/ipc/writer.c | 2 +- 11 files changed, 89 insertions(+), 47 deletions(-) diff --git a/src/nanoarrow/common/array.c b/src/nanoarrow/common/array.c index f9773c8f3..ee308ea61 100644 --- a/src/nanoarrow/common/array.c +++ b/src/nanoarrow/common/array.c @@ -854,7 +854,7 @@ ArrowErrorCode ArrowArrayAllocateChildren(struct ArrowArray* array, int64_t n_ch return ENOMEM; } - memset(array->children, 0, (size_t)n_children * sizeof(struct ArrowArray*)); + memset(array->children, 0, _ArrowInt64ToSizeT(n_children) * sizeof(struct ArrowArray*)); for (int64_t i = 0; i < n_children; i++) { array->children[i] = (struct ArrowArray*)ArrowMalloc(sizeof(struct ArrowArray)); @@ -2365,7 +2365,7 @@ static void ArrowArrayViewCompareBuffer(const struct ArrowArrayView* actual, int64_t size_bytes = actual->buffer_views[i].size_bytes; if (size_bytes > 0) { - size_t buffer_size = (size_t)size_bytes; + size_t buffer_size = _ArrowInt64ToSizeT(size_bytes); SET_NOT_EQUAL_AND_RETURN_IF( memcmp(actual->buffer_views[i].data.data, expected->buffer_views[i].data.data, buffer_size) != 0, diff --git a/src/nanoarrow/common/inline_array.h b/src/nanoarrow/common/inline_array.h index 5f75326f2..7153a3355 100644 --- a/src/nanoarrow/common/inline_array.h +++ b/src/nanoarrow/common/inline_array.h @@ -594,7 +594,8 @@ static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array, if (value.size_bytes <= NANOARROW_BINARY_VIEW_INLINE_SIZE) { if (value.size_bytes > 0) { - memcpy(bvt.inlined.data, value.data.as_char, (size_t)value.size_bytes); + memcpy(bvt.inlined.data, value.data.as_char, + _ArrowInt64ToSizeT(value.size_bytes)); } memset(bvt.inlined.data + bvt.inlined.size, 0, NANOARROW_BINARY_VIEW_INLINE_SIZE - bvt.inlined.size); diff --git a/src/nanoarrow/common/inline_buffer.h b/src/nanoarrow/common/inline_buffer.h index 0694264ec..b99aa5d71 100644 --- a/src/nanoarrow/common/inline_buffer.h +++ b/src/nanoarrow/common/inline_buffer.h @@ -247,7 +247,7 @@ static inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const voi int64_t size_bytes) { if (size_bytes > 0) { NANOARROW_DCHECK(buffer->data != NULL); - memcpy(buffer->data + buffer->size_bytes, data, (size_t)size_bytes); + memcpy(buffer->data + buffer->size_bytes, data, _ArrowInt64ToSizeT(size_bytes)); buffer->size_bytes += size_bytes; } } @@ -329,7 +329,7 @@ static inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer, NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes)); NANOARROW_DCHECK(buffer->data != NULL); // To help clang-tidy - memset(buffer->data + buffer->size_bytes, value, (size_t)size_bytes); + memset(buffer->data + buffer->size_bytes, value, _ArrowInt64ToSizeT(size_bytes)); buffer->size_bytes += size_bytes; return NANOARROW_OK; diff --git a/src/nanoarrow/common/inline_types.h b/src/nanoarrow/common/inline_types.h index 9b19d918b..5737d6012 100644 --- a/src/nanoarrow/common/inline_types.h +++ b/src/nanoarrow/common/inline_types.h @@ -327,6 +327,22 @@ static inline void ArrowErrorSetString(struct ArrowError* error, const char* src #define NANOARROW_DCHECK(EXPR) #endif +// Returns nonzero if x is a valid (non-negative) size_t value on this platform +static inline int _ArrowInt64FitsInSizeT(int64_t x) { +#if SIZE_MAX < INT64_MAX + return x >= 0 && x <= (int64_t)SIZE_MAX; +#else + return x >= 0; +#endif +} + +// Convert a size or offset known to fit in size_t (e.g., the size of an existing +// in-memory allocation). Checked in debug builds only. +static inline size_t _ArrowInt64ToSizeT(int64_t x) { + NANOARROW_DCHECK(_ArrowInt64FitsInSizeT(x)); + return (size_t)x; +} + static inline void ArrowSchemaMove(struct ArrowSchema* src, struct ArrowSchema* dst) { NANOARROW_DCHECK(src != NULL); NANOARROW_DCHECK(dst != NULL); diff --git a/src/nanoarrow/common/schema.c b/src/nanoarrow/common/schema.c index 997602a45..3f6120ceb 100644 --- a/src/nanoarrow/common/schema.c +++ b/src/nanoarrow/common/schema.c @@ -434,18 +434,18 @@ ArrowErrorCode ArrowSchemaSetTypeUnion(struct ArrowSchema* schema, enum ArrowTyp // Max valid size would be +ud:0,1,...126 = 401 characters + null terminator char format_out[512]; int64_t format_out_size = 512; - memset(format_out, 0, (size_t)format_out_size); + memset(format_out, 0, _ArrowInt64ToSizeT(format_out_size)); int n_chars; char* format_cursor = format_out; switch (type) { case NANOARROW_TYPE_SPARSE_UNION: - n_chars = snprintf(format_cursor, (size_t)format_out_size, "+us:"); + n_chars = snprintf(format_cursor, _ArrowInt64ToSizeT(format_out_size), "+us:"); format_cursor += n_chars; format_out_size -= n_chars; break; case NANOARROW_TYPE_DENSE_UNION: - n_chars = snprintf(format_cursor, (size_t)format_out_size, "+ud:"); + n_chars = snprintf(format_cursor, _ArrowInt64ToSizeT(format_out_size), "+ud:"); format_cursor += n_chars; format_out_size -= n_chars; break; @@ -460,12 +460,13 @@ ArrowErrorCode ArrowSchemaSetTypeUnion(struct ArrowSchema* schema, enum ArrowTyp } if (n_children > 0) { - n_chars = snprintf(format_cursor, (size_t)format_out_size, "0"); + n_chars = snprintf(format_cursor, _ArrowInt64ToSizeT(format_out_size), "0"); format_cursor += n_chars; format_out_size -= n_chars; for (int64_t i = 1; i < n_children; i++) { - n_chars = snprintf(format_cursor, (size_t)format_out_size, ",%" PRId64, i); + n_chars = + snprintf(format_cursor, _ArrowInt64ToSizeT(format_out_size), ",%" PRId64, i); format_cursor += n_chars; format_out_size -= n_chars; } @@ -533,7 +534,7 @@ ArrowErrorCode ArrowSchemaSetMetadata(struct ArrowSchema* schema, const char* me } if (metadata != NULL) { - size_t metadata_size = (size_t)ArrowMetadataSizeOf(metadata); + size_t metadata_size = _ArrowInt64ToSizeT(ArrowMetadataSizeOf(metadata)); schema->metadata = (const char*)ArrowMalloc(metadata_size); if (schema->metadata == NULL) { return ENOMEM; @@ -563,7 +564,8 @@ ArrowErrorCode ArrowSchemaAllocateChildren(struct ArrowSchema* schema, schema->n_children = n_children; - memset(schema->children, 0, (size_t)n_children * sizeof(struct ArrowSchema*)); + memset(schema->children, 0, + _ArrowInt64ToSizeT(n_children) * sizeof(struct ArrowSchema*)); for (int64_t i = 0; i < n_children; i++) { schema->children[i] = (struct ArrowSchema*)ArrowMalloc(sizeof(struct ArrowSchema)); @@ -1616,9 +1618,9 @@ static ArrowErrorCode ArrowMetadataGetValueInternal(const char* metadata, while (ArrowMetadataReaderRead(&reader, &existing_key, &existing_value) == NANOARROW_OK) { - int key_equal = - key->size_bytes == existing_key.size_bytes && - strncmp(key->data, existing_key.data, (size_t)existing_key.size_bytes) == 0; + int key_equal = key->size_bytes == existing_key.size_bytes && + strncmp(key->data, existing_key.data, + _ArrowInt64ToSizeT(existing_key.size_bytes)) == 0; if (key_equal) { value_out->data = existing_value.data; value_out->size_bytes = existing_value.size_bytes; @@ -1722,7 +1724,7 @@ static ArrowErrorCode ArrowMetadataBuilderSetInternal(struct ArrowBuffer* buffer if (key->size_bytes == existing_key.size_bytes && strncmp((const char*)key->data, (const char*)existing_key.data, - (size_t)existing_key.size_bytes) == 0) { + _ArrowInt64ToSizeT(existing_key.size_bytes)) == 0) { result = ArrowMetadataBuilderAppendInternal(&new_buffer, key, value); value = NULL; } else { diff --git a/src/nanoarrow/common/utils.c b/src/nanoarrow/common/utils.c index a2e2ca1ad..f0726bab0 100644 --- a/src/nanoarrow/common/utils.c +++ b/src/nanoarrow/common/utils.c @@ -237,9 +237,22 @@ void ArrowLayoutInit(struct ArrowLayout* layout, enum ArrowType storage_type) { } } -void* ArrowMalloc(int64_t size) { return malloc((size_t)size); } +void* ArrowMalloc(int64_t size) { + if (!_ArrowInt64FitsInSizeT(size)) { + return NULL; + } + + return malloc((size_t)size); +} -void* ArrowRealloc(void* ptr, int64_t size) { return realloc(ptr, (size_t)size); } +void* ArrowRealloc(void* ptr, int64_t size) { + if (!_ArrowInt64FitsInSizeT(size)) { + free(ptr); + return NULL; + } + + return realloc(ptr, (size_t)size); +} void ArrowFree(void* ptr) { free(ptr); } @@ -542,7 +555,7 @@ static void ShiftAndAdd(struct ArrowStringView value, uint32_t* out, int64_t out const uint64_t multiple = kUInt32PowersOfTen[group_size]; - memcpy(chunk_string, value.data + posn, (size_t)group_size); + memcpy(chunk_string, value.data + posn, _ArrowInt64ToSizeT(group_size)); chunk_string[group_size] = '\0'; uint32_t chunk = (uint32_t)strtoll(chunk_string, NULL, 10); @@ -627,9 +640,9 @@ ArrowErrorCode ArrowDecimalAppendDigitsToBuffer(const struct ArrowDecimal* decim int32_t value; memcpy(&value, decimal->words, sizeof(int32_t)); NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, 16)); - int n_chars = - snprintf((char*)buffer->data + buffer->size_bytes, - (size_t)(buffer->capacity_bytes - buffer->size_bytes), "%d", value); + int n_chars = snprintf( + (char*)buffer->data + buffer->size_bytes, + _ArrowInt64ToSizeT(buffer->capacity_bytes - buffer->size_bytes), "%d", value); if (n_chars <= 0) { return EINVAL; } @@ -784,7 +797,8 @@ ArrowErrorCode ArrowDecimalAppendStringToBuffer(const struct ArrowDecimal* decim ArrowBufferResize(buffer, buffer->size_bytes + num_zeros_after_decimal + 2, 0)); uint8_t* digits_start = buffer->data + is_negative; - memmove(digits_start + num_zeros_after_decimal + 2, digits_start, (size_t)num_digits); + memmove(digits_start + num_zeros_after_decimal + 2, digits_start, + _ArrowInt64ToSizeT(num_digits)); *digits_start++ = '0'; *digits_start++ = '.'; for (int i = 0; i < num_zeros_after_decimal; i++) { diff --git a/src/nanoarrow/device/device.c b/src/nanoarrow/device/device.c index 88a27ba7f..e4b7dbf81 100644 --- a/src/nanoarrow/device/device.c +++ b/src/nanoarrow/device/device.c @@ -89,7 +89,8 @@ static ArrowErrorCode ArrowDeviceCpuBufferCopy(struct ArrowDevice* device_src, return EINVAL; } - memcpy((uint8_t*)dst.data.as_uint8, src.data.as_uint8, (size_t)dst.size_bytes); + memcpy((uint8_t*)dst.data.as_uint8, src.data.as_uint8, + _ArrowInt64ToSizeT(dst.size_bytes)); return NANOARROW_OK; } diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 15d769650..e02390745 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -721,7 +721,7 @@ static ArrowErrorCode ArrowIpcDecoderVisitMetadata( int64_t n_pairs = ns(KeyValue_vec_len(kv_vec)); for (int64_t i = 0; i < n_pairs; i++) { - ns(KeyValue_table_t) kv = ns(KeyValue_vec_at(kv_vec, (size_t)i)); + ns(KeyValue_table_t) kv = ns(KeyValue_vec_at(kv_vec, _ArrowInt64ToSizeT(i))); flatbuffers_string_t key = ns(KeyValue_key(kv)); flatbuffers_string_t value = ns(KeyValue_value(kv)); @@ -1168,7 +1168,7 @@ static int ArrowIpcDecoderSetTypeUnion(struct ArrowSchema* schema, for (int64_t i = 1; i < n_type_ids; i++) { n_chars = snprintf(format_cursor, format_out_size, ",%" PRId32, - flatbuffers_int32_vec_at(type_ids, (size_t)i)); + flatbuffers_int32_vec_at(type_ids, _ArrowInt64ToSizeT(i))); format_cursor += n_chars; format_out_size -= n_chars; @@ -1417,7 +1417,7 @@ static int ArrowIpcDecoderSetChildren(struct ArrowSchema* schema, ns(Field_vec_t int64_t n_fields = ns(Schema_vec_len(fields)); for (int64_t i = 0; i < n_fields; i++) { - ns(Field_table_t) field = ns(Field_vec_at(fields, (size_t)i)); + ns(Field_table_t) field = ns(Field_vec_at(fields, _ArrowInt64ToSizeT(i))); NANOARROW_RETURN_NOT_OK( ArrowIpcDecoderSetField(schema->children[i], field, dictionaries, error)); } @@ -1449,7 +1449,7 @@ static int ArrowIpcDecoderDecodeSchemaHeader(struct ArrowIpcDecoder* decoder, decoder->feature_flags = 0; for (int64_t i = 0; i < n_features; i++) { - ns(Feature_enum_t) feature = ns(Feature_vec_at(features, (size_t)i)); + ns(Feature_enum_t) feature = ns(Feature_vec_at(features, _ArrowInt64ToSizeT(i))); switch (feature) { case ns(Feature_COMPRESSED_BODY): decoder->feature_flags |= NANOARROW_IPC_FEATURE_COMPRESSED_BODY; @@ -2087,7 +2087,7 @@ ArrowErrorCode ArrowIpcDecoderSetSchemaWithDictionaries( return ENOMEM; } memset(private_data->fields, 0, - (size_t)private_data->n_fields * sizeof(struct ArrowIpcField)); + _ArrowInt64ToSizeT(private_data->n_fields) * sizeof(struct ArrowIpcField)); // Init field information and calculate starting buffer offset for each int64_t field_i = 0; diff --git a/src/nanoarrow/ipc/encoder.c b/src/nanoarrow/ipc/encoder.c index 02577ac71..1a7c53751 100644 --- a/src/nanoarrow/ipc/encoder.c +++ b/src/nanoarrow/ipc/encoder.c @@ -227,8 +227,8 @@ void ArrowIpcEncoderSetDictionaryReplacement(struct ArrowIpcEncoder* encoder, } static ArrowErrorCode ArrowIpcEncoderWriteContinuationAndSize(struct ArrowBuffer* out, - size_t size) { - _NANOARROW_CHECK_UPPER_LIMIT(size, INT32_MAX); + int64_t size) { + _NANOARROW_CHECK_RANGE(size, 0, INT32_MAX); NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(out, -1)); if (ArrowIpcSystemEndianness() == NANOARROW_IPC_ENDIANNESS_BIG) { @@ -250,8 +250,7 @@ ArrowErrorCode ArrowIpcEncoderFinalizeBuffer(struct ArrowIpcEncoder* encoder, int64_t padded_size = _ArrowRoundUpToMultipleOf8(size); NANOARROW_RETURN_NOT_OK( ArrowBufferReserve(out, sizeof(int32_t) + sizeof(int32_t) + padded_size)); - NANOARROW_ASSERT_OK( - ArrowIpcEncoderWriteContinuationAndSize(out, (size_t)padded_size)); + NANOARROW_RETURN_NOT_OK(ArrowIpcEncoderWriteContinuationAndSize(out, padded_size)); } else { NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(out, size)); } @@ -504,10 +503,12 @@ static ArrowErrorCode ArrowIpcEncodeMetadata(flatcc_builder_t* builder, NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowMetadataReaderRead(&metadata, &key, &value), error); FLATCC_RETURN_UNLESS_0_NO_NS(push_start(builder), error); - FLATCC_RETURN_UNLESS_0(KeyValue_key_create(builder, key.data, (size_t)key.size_bytes), - error); FLATCC_RETURN_UNLESS_0( - KeyValue_value_create(builder, value.data, (size_t)value.size_bytes), error); + KeyValue_key_create(builder, key.data, _ArrowInt64ToSizeT(key.size_bytes)), + error); + FLATCC_RETURN_UNLESS_0( + KeyValue_value_create(builder, value.data, _ArrowInt64ToSizeT(value.size_bytes)), + error); FLATCC_RETURN_IF_NULL(push_end(builder), error); } return NANOARROW_OK; @@ -823,8 +824,9 @@ static ArrowErrorCode ArrowIpcEncoderReserveCompressedBuffers( } if (private->n_compressed_buffers > 0) { - memcpy(buffers, private->compressed_buffers, - (size_t) private->n_compressed_buffers * sizeof(struct ArrowBuffer)); + memcpy( + buffers, private->compressed_buffers, + _ArrowInt64ToSizeT(private->n_compressed_buffers) * sizeof(struct ArrowBuffer)); ArrowFree(private->compressed_buffers); } for (int64_t i = private->n_compressed_buffers; i < n; i++) { @@ -1079,12 +1081,12 @@ static ArrowErrorCode ArrowIpcEncoderEncodeRecordBatch( FLATCC_RETURN_UNLESS_0( RecordBatch_nodes_create( // builder, (struct ns(FieldNode)*)private->nodes.data, - (size_t) private->nodes.size_bytes / sizeof(struct ns(FieldNode))), + _ArrowInt64ToSizeT(private->nodes.size_bytes) / sizeof(struct ns(FieldNode))), error); FLATCC_RETURN_UNLESS_0( RecordBatch_buffers_create( // builder, (struct ns(Buffer)*)private->buffers.data, - (size_t) private->buffers.size_bytes / sizeof(struct ns(Buffer))), + _ArrowInt64ToSizeT(private->buffers.size_bytes) / sizeof(struct ns(Buffer))), error); FLATCC_RETURN_UNLESS_0(Message_header_RecordBatch_end(builder), error); @@ -1151,12 +1153,12 @@ static ArrowErrorCode ArrowIpcEncoderEncodeDictionaryBatch( FLATCC_RETURN_UNLESS_0( RecordBatch_nodes_create( builder, (struct ns(FieldNode)*)private->nodes.data, - (size_t) private->nodes.size_bytes / sizeof(struct ns(FieldNode))), + _ArrowInt64ToSizeT(private->nodes.size_bytes) / sizeof(struct ns(FieldNode))), error); FLATCC_RETURN_UNLESS_0( RecordBatch_buffers_create( builder, (struct ns(Buffer)*)private->buffers.data, - (size_t) private->buffers.size_bytes / sizeof(struct ns(Buffer))), + _ArrowInt64ToSizeT(private->buffers.size_bytes) / sizeof(struct ns(Buffer))), error); FLATCC_RETURN_UNLESS_0(DictionaryBatch_data_end(builder), error); FLATCC_RETURN_UNLESS_0(DictionaryBatch_isDelta_add(builder, is_delta ? 1 : 0), error); @@ -1230,7 +1232,7 @@ ArrowErrorCode ArrowIpcEncoderEncodeFooter(struct ArrowIpcEncoder* encoder, FLATCC_RETURN_UNLESS_0(Footer_recordBatches_start(builder), error); struct ns(Block)* flatcc_RecordBatch_blocks = - ns(Footer_recordBatches_extend(builder, (size_t)n_blocks)); + ns(Footer_recordBatches_extend(builder, _ArrowInt64ToSizeT(n_blocks))); FLATCC_RETURN_IF_NULL(flatcc_RecordBatch_blocks, error); for (int64_t i = 0; i < n_blocks; i++) { struct ns(Block) block = { @@ -1249,7 +1251,7 @@ ArrowErrorCode ArrowIpcEncoderEncodeFooter(struct ArrowIpcEncoder* encoder, FLATCC_RETURN_UNLESS_0(Footer_dictionaries_start(builder), error); struct ns(Block)* flatcc_dict_blocks = - ns(Footer_dictionaries_extend(builder, (size_t)n_dict_blocks)); + ns(Footer_dictionaries_extend(builder, _ArrowInt64ToSizeT(n_dict_blocks))); FLATCC_RETURN_IF_NULL(flatcc_dict_blocks, error); for (int64_t i = 0; i < n_dict_blocks; i++) { struct ns(Block) block = { diff --git a/src/nanoarrow/ipc/reader.c b/src/nanoarrow/ipc/reader.c index 8e6e5c6b3..6dc1a5285 100644 --- a/src/nanoarrow/ipc/reader.c +++ b/src/nanoarrow/ipc/reader.c @@ -67,7 +67,7 @@ static ArrowErrorCode ArrowIpcInputStreamBufferRead(struct ArrowIpcInputStream* if (bytes_to_read > 0) { memcpy(buf, private_data->input.data + private_data->cursor_bytes, - (size_t)bytes_to_read); + _ArrowInt64ToSizeT(bytes_to_read)); } *size_read_out = bytes_to_read; @@ -135,7 +135,7 @@ static ArrowErrorCode ArrowIpcInputStreamFileRead(struct ArrowIpcInputStream* st // Do the read int64_t bytes_read = - (int64_t)fread(buf, 1, (size_t)buf_size_bytes, private_data->file_ptr); + (int64_t)fread(buf, 1, _ArrowInt64ToSizeT(buf_size_bytes), private_data->file_ptr); *size_read_out = bytes_read; if (bytes_read != buf_size_bytes) { @@ -292,6 +292,12 @@ static int ArrowIpcArrayStreamReaderNextHeader( // https://github.com/apache/arrow-nanoarrow/issues/648 int64_t extra_bytes_already_read = 0; if (prefix_size_bytes == 4) { + if (!_ArrowInt64FitsInSizeT(private_data->header.size_bytes)) { + ArrowErrorSet(&private_data->error, "Header size %" PRId64 " is out of range", + private_data->header.size_bytes); + return EINVAL; + } + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowBufferReserve(&private_data->header, 4), &private_data->error); memmove(private_data->header.data + 4, private_data->header.data, diff --git a/src/nanoarrow/ipc/writer.c b/src/nanoarrow/ipc/writer.c index b0116d699..18252125c 100644 --- a/src/nanoarrow/ipc/writer.c +++ b/src/nanoarrow/ipc/writer.c @@ -124,7 +124,7 @@ static ArrowErrorCode ArrowIpcOutputStreamFileWrite(struct ArrowIpcOutputStream* // Do the write int64_t bytes_written = - (int64_t)fwrite(buf, 1, (size_t)buf_size_bytes, private_data->file_ptr); + (int64_t)fwrite(buf, 1, _ArrowInt64ToSizeT(buf_size_bytes), private_data->file_ptr); *size_written_out = bytes_written; if (bytes_written != buf_size_bytes) {