diff --git a/.github/scripts/build-research-recipes.sh b/.github/scripts/build-research-recipes.sh index 73b6ebf0c..4962b327c 100755 --- a/.github/scripts/build-research-recipes.sh +++ b/.github/scripts/build-research-recipes.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Compile owners concurrently. Keep each owner's recipes and target writes serial. set -euo pipefail -[[ $# == 2 || $# == 5 ]] || exit 2 +[[ $# == 2 || $# == 6 ]] || exit 2 product=${1:?product required} phase=${2:?recipe measurement phase required} [[ $phase == after-cache-lookup || $phase == warm-local || $phase == dependency-warm-serial || $phase == dependency-warm-parallel ]] || exit 2 @@ -58,7 +58,7 @@ if [[ $# == 2 ]]; then trap 'cancel_scheduler 143' TERM # Background execution lets Bash handle a signal sent only to its own PID. # The Ruby supervisor receives that signal and reaps its owned process groups. - ruby -rjson - "$root/.github/scripts/build-research-recipes.sh" "$product" "$phase" "$inputs" "$cpus" "$memory_mib" <<'RUBY' & + ruby -rjson -rtmpdir -rfileutils - "$root/.github/scripts/build-research-recipes.sh" "$product" "$phase" "$inputs" "$cpus" "$memory_mib" <<'RUBY' & script, product, phase, inputs, cpus, memory = ARGV cpus, memory = Integer(cpus), Integer(memory) requested = ENV.fetch('MONDAY_RESEARCH_BUILD_CONCURRENCY', '2') @@ -71,6 +71,8 @@ owners = JSON.parse(File.read(inputs)).fetch('recipes').map { |recipe| recipe.fe concurrency = [Integer(requested), owners.length, budget, [memory / 4096, 1].max].min jobs = budget / concurrency pending, active = owners.dup, {} +lease_root = Dir.mktmpdir('research-recipe-jobs.', ENV.fetch('RUNNER_TEMP')) +leases = {} failed_groups = [] status = 0 interrupted = nil @@ -85,13 +87,25 @@ begin until pending.empty? && active.empty? while !interrupted && active.length < concurrency && !pending.empty? owner = pending.shift - pid = Process.spawn('bash', script, product, phase, owner, jobs.to_s, concurrency.to_s, pgroup: true) + lease = File.join(lease_root, owner.delete_suffix('/Cargo.toml').tr('/', '-') + '.jobs') + File.write(lease, jobs.to_s) + pid = Process.spawn('bash', script, product, phase, owner, jobs.to_s, concurrency.to_s, lease, pgroup: true) active[pid] = owner + leases[pid] = lease end if interrupted status = 128 + Signal.list.fetch(interrupted) break end + # Increase only the final owner's next recipe. No pending or sibling owner + # can claim these CPUs. An in-flight Cargo/cc jobserver keeps its old budget. + if pending.empty? && active.length == 1 + lease = leases.fetch(active.keys.first) + unless File.read(lease) == budget.to_s + File.write(lease + '.next', budget.to_s) + File.rename(lease + '.next', lease) + end + end completed = Process.wait2(-1, Process::WNOHANG) unless completed IO.select([wake_read]) @@ -114,6 +128,7 @@ ensure sleep 0.2 unless groups.empty? groups.each { |pid| Process.kill('KILL', -pid) rescue Errno::ESRCH } groups.each { |pid| Process.wait(pid) rescue Errno::ECHILD } + FileUtils.remove_entry(lease_root) wake_read.close wake_write.close end @@ -147,31 +162,42 @@ RUBY [[ $(sha256sum "$inputs" | awk '{print $1}') == "$inputs_sha" ]] exit "$status" fi -manifest=$3 cargo_jobs=$4 concurrency=$5 +manifest=$3 cargo_jobs=$4 concurrency=$5 lease=$6 [[ $cargo_jobs =~ ^[1-9][0-9]*$ && $concurrency =~ ^[12]$ ]] (( cargo_jobs * concurrency <= cpus )) jq -e --arg manifest "$manifest" 'any(.[]; .manifest==$manifest)' <<<"$recipes" >/dev/null owner=${manifest%/Cargo.toml} timings="$work/$phase-${owner//\//--}.timings.jsonl" while IFS= read -r recipe; do + # The supervisor grants a larger budget only after all other owners exit. + # Read once per recipe; never change Cargo's budget during compilation. + next_jobs=$(cat "$lease") + [[ $next_jobs =~ ^[1-9][0-9]*$ ]] + (( next_jobs >= cargo_jobs && next_jobs <= cpus )) + cargo_jobs=$next_jobs package=$(jq -r .package <<<"$recipe") - args=(build --manifest-path "$manifest" --target x86_64-unknown-linux-gnu --release --locked --jobs "$cargo_jobs" -p "$package") - features=$(jq -r .features <<<"$recipe") - [[ -z $features ]] || args+=(--features "$features") - while IFS= read -r binary; do args+=(--bin "$binary"); done < <(jq -r '.binaries[]' <<<"$recipe") start=$(date +%s) recipe_target=$(bash "$root/.github/scripts/research-cache-layout.sh" target-dir "$(jq -r .manifest <<<"$recipe")") - CARGO_TARGET_DIR="$recipe_target" cargo "${args[@]}" --message-format json-render-diagnostics >"$work/$phase-$package.jsonl" + # Cargo emits dirty reasons to stderr; keep stdout as compiler JSON only. + # Logging belongs to orchestration, not the dependency compilation identity. + fingerprint_log="$work/$phase-$package.fingerprint.log" + status=0 + { CARGO_LOG=cargo::core::compiler::fingerprint=info CARGO_TERM_COLOR=never \ + CARGO_TARGET_DIR="$recipe_target" bash "$root/.github/scripts/compile-research-recipe.sh" "$recipe" "$cargo_jobs" \ + >"$work/$phase-$package.jsonl"; } 2>&1 | tee "$fingerprint_log" >&2 || status=$? + (( status == 0 )) || exit "$status" jq -es 'any(.[]; .reason=="compiler-artifact") and any(.[]; .reason=="build-finished" and .success==true)' \ "$work/$phase-$package.jsonl" >/dev/null seconds=$(( $(date +%s) - start )) jq -s -c --arg cache_match "$cache_match" --arg phase "$phase" --arg source "$source_sha" --arg inputs "$inputs_sha" \ --argjson requested_concurrency "${MONDAY_RESEARCH_BUILD_CONCURRENCY:-2}" \ --argjson cargo_jobs "$cargo_jobs" --argjson concurrency "$concurrency" \ + --rawfile diagnostics "$fingerprint_log" \ --argjson runner "$runner_context" --arg recipe_target "$recipe_target" --argjson recipe "$recipe" --argjson seconds "$seconds" \ '{phase:$phase,runner:$runner,recipe_target:$recipe_target,cache_exact_match:$cache_match,source_sha:$source,compilation_inputs_sha256:$inputs,recipe:$recipe,seconds:$seconds, scheduler:{requested_owner_concurrency:$requested_concurrency,owner_concurrency:$concurrency,cargo_jobs:$cargo_jobs}, compiler_artifacts:([.[]|select(.reason=="compiler-artifact")]|length), + native_fingerprint_diagnostics:[$diagnostics|split("\n")[]|select(contains("fingerprint") and contains("libduckdb"))], native_artifacts:[.[]|select(.reason=="compiler-artifact" and (.package_id|contains("libduckdb-sys")))|{package_id,target:.target.name,fresh}], fresh:([.[]|select(.reason=="compiler-artifact" and .fresh==true)]|length), rebuilt:([.[]|select(.reason=="compiler-artifact" and .fresh==false)]|length)}' \ diff --git a/.github/scripts/capture-research-build-inputs.sh b/.github/scripts/capture-research-build-inputs.sh index ffcc600f4..3e8696be4 100755 --- a/.github/scripts/capture-research-build-inputs.sh +++ b/.github/scripts/capture-research-build-inputs.sh @@ -4,6 +4,7 @@ set -euo pipefail output=${1:?output JSON required} root=$(cd "$(dirname "$0")/../.." && pwd) +bash "$root/.github/scripts/prepare-research-native-source.sh" work=$(mktemp -d) trap 'rm -rf "$work"' EXIT test "$(uname -s)" = Linux @@ -29,9 +30,19 @@ for variable in CC CXX AR LD CFLAGS CXXFLAGS LDFLAGS RUSTC_BOOTSTRAP CARGO_BUILD LIBRARY_PATH LD_LIBRARY_PATH CPATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH \ PKG_CONFIG PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR \ OPENSSL_DIR OPENSSL_LIB_DIR OPENSSL_INCLUDE_DIR OPENSSL_STATIC \ - BINDGEN_EXTRA_CLANG_ARGS PROTOC PROTOC_INCLUDE CMAKE_TOOLCHAIN_FILE; do + BINDGEN_EXTRA_CLANG_ARGS PROTOC PROTOC_INCLUDE CMAKE_TOOLCHAIN_FILE \ + DUCKDB_STATIC DUCKDB_LIB_DIR DUCKDB_INCLUDE_DIR DUCKDB_DOWNLOAD_LIB \ + RANLIB ARFLAGS CXXSTDLIB CRATE_CC_NO_DEFAULTS CC_SHELL_ESCAPED_FLAGS \ + CMAKE_GENERATOR CMAKE_MAKE_PROGRAM CMAKE_PREFIX_PATH; do printf '%s=%s\n' "$variable" "${!variable-}" >>"$work/native" done +# cc-rs also accepts target-specific compiler and flag overrides. +while IFS= read -r variable; do + printf '%s=%s\n' "$variable" "$(printenv "$variable")" >>"$work/native" +done < <(env | awk -F= '$1 ~ /^(CC|CXX|AR|RANLIB|CFLAGS|CXXFLAGS|ARFLAGS|CXXSTDLIB)_(x86_64|x86-64)/ || $1 ~ /^(HOST|TARGET)_(CC|CXX|AR|RANLIB|CFLAGS|CXXFLAGS|ARFLAGS|CXXSTDLIB)$/ {print $1}' | LC_ALL=C sort) +for command in cc c++ ar ld ranlib; do + sha256sum "$(command -v "$command")" >>"$work/native" +done for command in clang cmake mold pkg-config protoc zstd zstdmt unzstd; do "$command" --version >>"$work/native"; done # Bind relevant Cargo overrides without capturing registry or service secrets. for variable in RUSTC RUSTDOC RUSTC_WORKSPACE_WRAPPER CARGO_INCREMENTAL; do @@ -62,17 +73,9 @@ while IFS= read -r manifest; do done < <(jq -r '.workspaces[].manifest' "$root/rust_hft/workspaces.json") recipes=$(bash "$root/.github/scripts/research-release-products.sh" recipes "${2:-all}" | jq -s .) locks=$("$root/.github/scripts/research-workspace-locks.sh" "$root/rust_hft") -cat "$root/.github/scripts/build-research-release.sh" \ - "$root/.github/scripts/build-research-recipes.sh" \ - "$root/.github/scripts/research-cache-layout.sh" \ - "$root/.github/scripts/research-release-products.sh" \ - "$root/.github/scripts/research-release-products.json" \ - "$root/.github/scripts/research-release-source-sha.sh" \ - "$root/.github/scripts/research-workspace-locks.sh" \ - "$root/.github/scripts/verify-research-runtime-abi.sh" \ - "$root/.github/scripts/verify-research-runner-binaries.sh" >"$work/recipe" -# Bind the fixed CI parser implementation alongside the build recipe. -find "$root/.github/scripts/vendor/tomlrb/lib" -type f -name '*.rb' -print0 | LC_ALL=C sort -z | xargs -0 cat >>"$work/recipe" +# The selected recipes below bind catalog contents to exact provenance. +# Packaging-only catalog edits must not invalidate compatible dependencies. +bash "$root/.github/scripts/research-cache-layout.sh" recipe-inputs >"$work/recipe" jq -S -n --arg compiler "$(sha256sum "$work/compiler" | awk '{print $1}')" \ --arg native "$(sha256sum "$work/native" | awk '{print $1}')" \ --arg flags "$(sha256sum "$work/flags" | awk '{print $1}')" \ @@ -86,6 +89,8 @@ jq -S -n --arg compiler "$(sha256sum "$work/compiler" | awk '{print $1}')" \ if [[ -n ${GITHUB_OUTPUT:-} ]]; then printf 'cache_sha256=%s\n' "$(sha256sum "$output" | awk '{print $1}')" >>"$GITHUB_OUTPUT"; fi # This separate restore identity does not replace the complete v3 provenance. bash "$root/.github/scripts/research-cache-layout.sh" compatibility-inputs "$output" >"$work/dependency-cache-inputs.json" +# Retain a small, public native identity summary in the producer log. +jq -c '.native_dependencies.closures[] | {owner,duckdb:[.packages[] | select(.name=="libduckdb-sys") | {name,version,checksum,features}]}' "$work/dependency-cache-inputs.json" if [[ -n ${GITHUB_OUTPUT:-} ]]; then printf 'dependency_cache_sha256=%s\n' "$(sha256sum "$work/dependency-cache-inputs.json" | awk '{print $1}')" >>"$GITHUB_OUTPUT" fi diff --git a/.github/scripts/ci-owner-cache.sh b/.github/scripts/ci-owner-cache.sh index fb1b1d6e8..accf1dd7b 100644 --- a/.github/scripts/ci-owner-cache.sh +++ b/.github/scripts/ci-owner-cache.sh @@ -130,14 +130,18 @@ begin "rust_hft/workspaces.json", ".github/scripts/ci-owner-cache.sh", ".github/scripts/test-rust-workspaces.sh", ".github/scripts/check-collector-test-presence.sh", ".github/workflows/ci.yml", ".github/scripts/select-rust-ci-scope.sh", - "deployment/aliyun/test-polymarket-raw-ops-control-plane.sh"] + "deployment/aliyun/test-polymarket-raw-ops-control-plane.sh", ".github/scripts/prepare-research-native-source.sh"] parser = Dir.glob("#{root}/.github/scripts/vendor/tomlrb/lib/**/*.rb").sort.to_h do |path| [path.delete_prefix("#{root}/"), file_digest(path)] end abort 'missing Cargo TOML parser' if parser.empty? - compatibility = {schema: 'monday.ci-owner-cache-compat.v2', compiler: compiler, target: host, + native_root = "#{root}/rust_hft/native/libduckdb-sys" + native_files = Dir.glob("#{native_root}/**/*", File::FNM_DOTMATCH).select { |path| File.file?(path) }.sort + abort 'native patch contains symlinks' if native_files.any? { |path| File.symlink?(path) } + native_patch = native_files.to_h { |path| [path.delete_prefix(native_root + '/'), file_digest(path)] } + compatibility = {native_patch: native_patch, native_preparation: file_digest("#{root}/.github/scripts/prepare-research-native-source.sh"), schema: 'monday.ci-owner-cache-compat.v3', compiler: compiler, target: host, profile: 'dev-test', native: native, flags: flags, configs: configs, profiles: profiles, owners: admitted, parser: parser} - prefix = "rust_hft-ci-owner-v2-#{digest(compatibility)}-" + prefix = "rust_hft-ci-owner-v3-#{digest(compatibility)}-" exact = {coverage: coverage, manifests: manifests, configs: configs, helpers: helpers.to_h { |path| [path, file_digest("#{root}/#{path}")] }} puts "compat_prefix=#{prefix}" @@ -188,7 +192,8 @@ if [[ $mode == native-input ]]; then CPATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH PKG_CONFIG PKG_CONFIG_PATH \ PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR BINDGEN_EXTRA_CLANG_ARGS \ OPENSSL_DIR OPENSSL_LIB_DIR OPENSSL_INCLUDE_DIR OPENSSL_STATIC \ - PROTOC PROTOC_INCLUDE CMAKE_TOOLCHAIN_FILE; do + PROTOC PROTOC_INCLUDE CMAKE_TOOLCHAIN_FILE \ + DUCKDB_STATIC DUCKDB_LIB_DIR DUCKDB_INCLUDE_DIR DUCKDB_DOWNLOAD_LIB; do printf '%s=%s\n' "$variable" "${!variable-}" done } >"$work/native" @@ -205,11 +210,11 @@ while IFS= read -r manifest; do --locked --all-features --format-version 1 >>"$work/metadata.jsonl" done <<<"$manifests" jq -se --arg root "$root/" \ - '[.[].packages[]|select(.source == null or (.manifest_path|startswith($root)))|.name,.targets[].name]|unique| + '[.[].packages[]|select((.source == null or (.manifest_path|startswith($root))) and (.manifest_path != ($root+"rust_hft/native/libduckdb-sys/Cargo.toml")))|.name,.targets[].name]|unique| if length>0 then . else error("no repository-local compilation packages") end' \ "$work/metadata.jsonl" >"$work/local-names.json" jq -s --arg root "$root/" \ - '[.[].packages[]|select(.source != null and (.manifest_path|startswith($root)|not))|.name,.targets[].name]|unique' \ + '[.[].packages[]|select((.source != null and (.manifest_path|startswith($root)|not)) or (.name == "libduckdb-sys" and .manifest_path == ($root+"rust_hft/native/libduckdb-sys/Cargo.toml")))|.name,.targets[].name]|unique' \ "$work/metadata.jsonl" >"$work/external-names.json" ruby -rjson -rfileutils - "$root" "$work/local-names.json" "$work/external-names.json" <<'RUBY' root, names_file, external_file = ARGV diff --git a/.github/scripts/compile-research-recipe.sh b/.github/scripts/compile-research-recipe.sh new file mode 100644 index 000000000..c4e881703 --- /dev/null +++ b/.github/scripts/compile-research-recipe.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Stable dependency compilation contract. Packaging and scheduling live outside it. +set -euo pipefail +recipe=${1:?recipe JSON required} +jobs=${2:?Cargo jobs required} +[[ $jobs =~ ^[1-9][0-9]*$ ]] +manifest=$(jq -er .manifest <<<"$recipe") +package=$(jq -er .package <<<"$recipe") +features=$(jq -er .features <<<"$recipe") +args=(build --manifest-path "$manifest" --target x86_64-unknown-linux-gnu --release --locked --jobs "$jobs" -p "$package") +[[ -z $features ]] || args+=(--features "$features") +while IFS= read -r binary; do args+=(--bin "$binary"); done < <(jq -er '.binaries[]' <<<"$recipe") +cargo "${args[@]}" --message-format json-render-diagnostics diff --git a/.github/scripts/prepare-research-native-source.sh b/.github/scripts/prepare-research-native-source.sh new file mode 100755 index 000000000..ac9f02e02 --- /dev/null +++ b/.github/scripts/prepare-research-native-source.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Cache lookup binds all native bytes. Normalize only a fresh CI source tree. +set -euo pipefail +root=$(cd "$(dirname "$0")/../.." && pwd) +[[ ${GITHUB_ACTIONS:-false} == true ]] || exit 0 +cd "$root" +source=rust_hft/native/libduckdb-sys +[[ -z $(git -c safe.directory="$root" status --porcelain --untracked-files=all -- "$source") ]] || { + echo 'native source must be committed and clean in CI' >&2; exit 1; +} +# Never hide an input change from an existing local or restored Cargo target. +for target in rust_hft/target rust_hft/research-core/target rust_hft/data-pipelines/target; do + [[ ! -e $target ]] || { echo "native timestamp preparation must precede target creation: $target" >&2; exit 1; } +done +ruby -ropen3 - "$source" <<'RUBY' +files, status = Open3.capture2('git', '-c', "safe.directory=#{Dir.pwd}", 'ls-files', '-z', '--', ARGV.fetch(0)) +abort 'native source inventory failed' unless status.success? && !files.empty? +files.split("\0").each do |file| + abort 'native source must contain regular files' unless File.file?(file) && !File.symlink?(file) + File.utime(Time.at(946684800), Time.at(946684800), file) +end +RUBY diff --git a/.github/scripts/research-cache-layout.sh b/.github/scripts/research-cache-layout.sh index d0cc44229..4a47f7452 100644 --- a/.github/scripts/research-cache-layout.sh +++ b/.github/scripts/research-cache-layout.sh @@ -38,6 +38,16 @@ validate_recipes() { (.binaries|type == "array" and length>0 and all(.[];name)))' "$1" >/dev/null } case ${1:?command required} in + recipe-inputs) + # Bind compiler orchestration and cleanup, independently of product selection. + # Exact selected recipes remain in compilation inputs and artifact admission. + for script in build-research-release.sh build-research-recipes.sh capture-research-build-inputs.sh research-cache-layout.sh \ + research-release-products.sh research-release-source-sha.sh research-workspace-locks.sh \ + verify-research-runtime-abi.sh verify-research-runner-binaries.sh research-native-dependency-inputs.rb compile-research-recipe.sh prepare-research-native-source.sh; do + cat "$root/.github/scripts/$script" + done + find "$root/.github/scripts/vendor/tomlrb/lib" -type f -name '*.rb' -print0 | LC_ALL=C sort -z | xargs -0 cat + ;; compatibility-inputs) # Keep local dependency choices in provenance and Cargo's exact cache suffix. # The restore prefix still binds every admitted compiler, native and profile input. @@ -124,9 +134,12 @@ rescue StandardError => error exit 1 end RUBY - jq -Se --slurpfile owner_profiles "$work/owner-profiles.json" ' - {schema:"monday.dependency-cache-compat.v3",target,profile,compiler,native,flags, - builder_image,recipe,owner_profiles:$owner_profiles[0]}' "$work/validated.json" + ruby "$root/.github/scripts/research-native-dependency-inputs.rb" "$root" "$work/cache-recipes.json" >"$work/native-dependencies.json" + dependency_recipe=$(cat "$root/.github/scripts/compile-research-recipe.sh" "$root/.github/scripts/research-cache-layout.sh" \ + "$root/.github/scripts/research-native-dependency-inputs.rb" "$root/.github/scripts/prepare-research-native-source.sh" | sha256sum | awk '{print $1}') + jq -Se --arg dependency_recipe "$dependency_recipe" --slurpfile native_dependencies "$work/native-dependencies.json" --slurpfile owner_profiles "$work/owner-profiles.json" ' + {schema:"monday.dependency-cache-compat.v4",target,profile,compiler,native,flags, + builder_image,dependency_recipe:$dependency_recipe,owner_profiles:$owner_profiles[0],native_dependencies:$native_dependencies[0]}' "$work/validated.json" ;; manifest-inputs) # Include local path/patch/default-feature manifests beyond recipe roots. @@ -198,8 +211,10 @@ RUBY fi cargo metadata --manifest-path "$root/rust_hft/$manifest" --locked --format-version 1 "${metadata_args[@]}" >>"$work/local.jsonl" done <<<"$recipes" - jq -s --arg root "$root/" '[.[].packages[]|select(.source == null or (.manifest_path|startswith($root)))|.name, .targets[].name]|unique|if length>0 then . else error("no local compilation packages") end' "$work/local.jsonl" >"$work/names.json" - jq -s --arg root "$root/" '[.[].packages[]|select(.source != null and (.manifest_path|startswith($root)|not))|.name, .targets[].name]|unique' "$work/local.jsonl" >"$work/external.json" + # The reviewed native patch is a dependency. Its complete source enters the + # native compatibility identity; application path packages remain excluded. + jq -s --arg root "$root/" '[.[].packages[]|select((.source == null or (.manifest_path|startswith($root))) and (.manifest_path != ($root+"rust_hft/native/libduckdb-sys/Cargo.toml")))|.name, .targets[].name]|unique|if length>0 then . else error("no local compilation packages") end' "$work/local.jsonl" >"$work/names.json" + jq -s --arg root "$root/" '[.[].packages[]|select((.source != null and (.manifest_path|startswith($root)|not)) or (.name == "libduckdb-sys" and .manifest_path == ($root+"rust_hft/native/libduckdb-sys/Cargo.toml")))|.name, .targets[].name]|unique' "$work/local.jsonl" >"$work/external.json" ruby -rjson -rfileutils - "$root" "$work/names.json" "$work/external.json" "$work/cache-recipes.json" <<'RUBY' root, names_file, external_file, recipes_file = ARGV names = JSON.parse(File.read(names_file)).flat_map { |n| [n, n.tr('-', '_'), "lib#{n.tr('-', '_')}"] }.uniq diff --git a/.github/scripts/research-native-dependency-inputs.rb b/.github/scripts/research-native-dependency-inputs.rb new file mode 100644 index 000000000..81e5cda30 --- /dev/null +++ b/.github/scripts/research-native-dependency-inputs.rb @@ -0,0 +1,63 @@ +#!/usr/bin/env ruby +# Cargo resolves native features. This identity excludes application source and packaging scripts. +require 'json' +require 'open3' +require 'digest' + +root, recipes_file = ARGV +abort 'root and recipes required' unless root && recipes_file +$LOAD_PATH.unshift("#{root}/.github/scripts/vendor/tomlrb/lib") +require 'tomlrb' +# Cargo metadata resolves the workspace. Reuse only identical command arguments. +# Keep feature selections separate; never union them across recipe invocations. +resolutions = {} +records = JSON.parse(File.read(recipes_file)).flat_map do |recipe| + manifest = recipe.fetch('manifest') + args = ['cargo', 'metadata', '--manifest-path', "#{root}/rust_hft/#{manifest}", + '--locked', '--format-version', '1', '--filter-platform', 'x86_64-unknown-linux-gnu'] + lock = Tomlrb.load_file("#{root}/rust_hft/#{File.dirname(manifest)}/Cargo.lock").fetch('package') + features = recipe.fetch('features') + unless features.empty? + args += ['--features', features.split(',').map { |feature| "#{recipe.fetch('package')}/#{feature}" }.join(',')] + end + metadata = resolutions[args] ||= begin + output, status = Open3.capture2(*args, chdir: "#{root}/rust_hft") + abort "native dependency resolution failed: #{manifest}" unless status.success? + JSON.parse(output) + end + packages = metadata.fetch('packages').to_h { |package| [package.fetch('id'), package] } + nodes = metadata.fetch('resolve').fetch('nodes').to_h { |node| [node.fetch('id'), node] } + native_roots = packages.values.select { |package| package.fetch('name') == 'libduckdb-sys' } + native_roots.map do |native| + pending = [native.fetch('id')] + visited = {} + until pending.empty? + id = pending.shift + next if visited.key?(id) + package = packages.fetch(id) + node = nodes.fetch(id) + source = package['source'] + checksum = nil + if source&.start_with?('registry+') + entry = lock.find { |item| item['name'] == package['name'] && item['version'] == package['version'] && item['source'] == source } + checksum = entry&.fetch('checksum', nil) + abort 'native registry source has no checksum' unless checksum&.match?(/\A[0-9a-f]{64}\z/) + end + # Registry versions and git revisions are immutable Cargo identities. + # Local overrides also bind source bytes, including the native build script. + local = nil + unless source + directory = File.dirname(package.fetch('manifest_path')) + files = Dir.glob("#{directory}/**/*", File::FNM_DOTMATCH).select { |file| File.file?(file) } + abort 'native local dependency contains a symlink' if files.any? { |file| File.symlink?(file) } + local = files.sort.to_h { |file| [file.delete_prefix(directory + '/'), Digest::SHA256.file(file).hexdigest] } + end + visited[id] = { 'name' => package.fetch('name'), 'version' => package.fetch('version'), + 'source' => source, 'checksum' => checksum, + 'features' => node.fetch('features').sort, 'local' => local } + pending.concat(node.fetch('dependencies')) + end + { 'owner' => manifest, 'packages' => visited.values.sort_by { |package| JSON.generate(package) } } + end +end +puts JSON.generate({ 'schema' => 'monday.native-dependencies.v1', 'closures' => records.uniq.sort_by { |record| JSON.generate(record) } }) diff --git a/.github/scripts/research-release-products.json b/.github/scripts/research-release-products.json index 7e09900ed..3f5588a32 100644 --- a/.github/scripts/research-release-products.json +++ b/.github/scripts/research-release-products.json @@ -8,8 +8,7 @@ "binance-market-tape-slicer", "binance-replay-parquet-materializer", "clickhouse-analytics-materializer", - "monday-cex-prepare", - "monday-offline-baseline" + "monday-cex-prepare" ], "prediction-runner": [ "monday-prediction-research", @@ -39,14 +38,6 @@ "alpha-harness" ] }, - { - "manifest": "research-core/Cargo.toml", - "package": "alpha-harness", - "features": "offline-diagnostic", - "binaries": [ - "monday-offline-baseline" - ] - }, { "manifest": "research-core/Cargo.toml", "package": "hft-cex-research-worker", diff --git a/.github/scripts/select-rust-ci-scope.sh b/.github/scripts/select-rust-ci-scope.sh index 142e05578..222800062 100755 --- a/.github/scripts/select-rust-ci-scope.sh +++ b/.github/scripts/select-rust-ci-scope.sh @@ -660,7 +660,7 @@ for path in "${paths[@]}"; do select_job ploy/workflow-lint continue ;; - .github/scripts/research-release-products.sh|.github/scripts/research-release-products.json|.github/scripts/test-research-release-products.sh|.github/scripts/research-release-bundle.rb|.github/scripts/research-release-source-sha.sh|.github/scripts/test-research-checkout-ownership.sh|.github/scripts/verify-research-runtime-abi.sh|.github/scripts/test-research-runtime-abi.sh|.github/scripts/research-cache-layout.sh|.github/scripts/test-research-cache-layout.sh|.github/scripts/build-research-release.sh|.github/scripts/build-research-recipes.sh|.github/scripts/capture-research-build-inputs.sh|.github/scripts/research-image-smoke.sh|.github/scripts/verify-research-product-image.sh|.github/scripts/test-research-product-image.sh|.github/scripts/verify-research-controller-image.sh|.github/scripts/test-research-controller-image.sh|.github/scripts/download-research-release.sh|.github/scripts/test-download-research-release.sh) + rust_hft/native/libduckdb-sys/*|.github/scripts/prepare-research-native-source.sh|.github/scripts/test-duckdb-build-modes.sh|.github/scripts/research-release-products.sh|.github/scripts/research-release-products.json|.github/scripts/test-research-release-products.sh|.github/scripts/research-release-bundle.rb|.github/scripts/research-release-source-sha.sh|.github/scripts/test-research-checkout-ownership.sh|.github/scripts/verify-research-runtime-abi.sh|.github/scripts/test-research-runtime-abi.sh|.github/scripts/research-cache-layout.sh|.github/scripts/test-research-cache-layout.sh|.github/scripts/build-research-release.sh|.github/scripts/build-research-recipes.sh|.github/scripts/compile-research-recipe.sh|.github/scripts/research-native-dependency-inputs.rb|.github/scripts/capture-research-build-inputs.sh|.github/scripts/research-image-smoke.sh|.github/scripts/verify-research-product-image.sh|.github/scripts/test-research-product-image.sh|.github/scripts/verify-research-controller-image.sh|.github/scripts/test-research-controller-image.sh|.github/scripts/download-research-release.sh|.github/scripts/test-download-research-release.sh) select_research_image_jobs select_job ci/ci-contracts select_job ploy/workflow-lint diff --git a/.github/scripts/test-ci-owner-cache.sh b/.github/scripts/test-ci-owner-cache.sh index 35e601690..d2a797a85 100644 --- a/.github/scripts/test-ci-owner-cache.sh +++ b/.github/scripts/test-ci-owner-cache.sh @@ -52,6 +52,8 @@ fi native_before=$(PATH="$work/native-bin:$PATH" bash "$root/.github/scripts/ci-owner-cache.sh" native-input) native_after=$(PATH="$work/native-bin:$PATH" CXXFLAGS=-DFIXTURE_NATIVE_INPUT_CHANGED bash "$root/.github/scripts/ci-owner-cache.sh" native-input) [[ $native_before != "$native_after" ]] +native_duckdb=$(PATH="$work/native-bin:$PATH" DUCKDB_LIB_DIR=/fixture/other-duckdb bash "$root/.github/scripts/ci-owner-cache.sh" native-input) +[[ $native_before != "$native_duckdb" ]] # A narrow first save must not become an exact hit for wider coverage. plan=$(jq -cn '{handoff:"false",json:"false",ondo:"false",collector:"false",control:"false",focused:"true",loop:"false",owning_packages:",,",focused_packages:",hft-live,",loop_packages:""}') coverage() { MONDAY_CI_CACHE_PLAN="$1" bash "$root/.github/scripts/ci-owner-cache.sh" coverage-input; } @@ -98,7 +100,7 @@ if [[ $1 == metadata ]]; then {id:"hft-data",name:"hft-data",manifest_path:($root+"/rust_hft/data-pipelines/core/Cargo.toml"),targets:[{name:"data"}]}] + if $extra then [{id:"vendor",name:"hidden-local",manifest_path:($root+"/vendor/hidden/Cargo.toml"),targets:[{name:"hidden_local"}]}, {id:"outside",name:"external-local",source:null,manifest_path:"/external/path/Cargo.toml",targets:[{name:"external_local"}]}, - {id:"registry",name:"libduckdb-sys",source:"registry+https://github.com/rust-lang/crates.io-index",manifest_path:"/registry/libduckdb/Cargo.toml",targets:[{name:"libduckdb_sys"}]}, + {id:"registry",name:"libduckdb-sys",source:null,manifest_path:($root+"/rust_hft/native/libduckdb-sys/Cargo.toml"),targets:[{name:"libduckdb_sys"}]}, {id:"encoding",name:"data-encoding",source:"registry+https://github.com/rust-lang/crates.io-index",manifest_path:"/registry/data-encoding/Cargo.toml",targets:[{name:"data_encoding"}]}] else [] end)}' else [[ ${FAIL_CARGO:-0} != 1 ]] || exit 19 @@ -312,6 +314,11 @@ Dir.mktmpdir('ci-cache-coverage') do |fixture| _, _, status = Open3.capture3('git', '-C', fixture, 'add', '.') abort 'fixture git add failed' unless status.success? baseline = cache_inputs.call(fixture) + native_file = "#{fixture}/rust_hft/native/libduckdb-sys/build.rs" + FileUtils.mkdir_p(File.dirname(native_file)) + File.write(native_file, 'changed native source fixture') + abort 'native source bytes crossed compatibility boundary' if cache_inputs.call(fixture)['compat_prefix'] == baseline['compat_prefix'] + FileUtils.rm_f(native_file) wide = cache_inputs.call(fixture, plan.merge('focused_packages' => 'hft-live,hft-cex-research-worker')) abort 'coverage changed compatible restore prefix' unless wide['compat_prefix'] == baseline['compat_prefix'] abort 'coverage changed no exact key' if wide['key'] == baseline['key'] diff --git a/.github/scripts/test-duckdb-build-modes.sh b/.github/scripts/test-duckdb-build-modes.sh new file mode 100755 index 000000000..a1a326d5e --- /dev/null +++ b/.github/scripts/test-duckdb-build-modes.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Run the patched build script in each mode. Bundled native linking remains in +# the image producer and offline smoke, with their admitted compiler and ABI. +set -euo pipefail +root=$(cd "$(dirname "$0")/../.." && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +mkdir -p "$work/src" +cat >"$work/Cargo.toml" <"$work/src/main.rs" +# Reuse the admitted dependency versions. Do not resolve a new latest baseline. +cp "$root/rust_hft/research-core/Cargo.lock" "$work/Cargo.lock" +export CARGO_TARGET_DIR="${MONDAY_NATIVE_TEST_TARGET:-$work/target}" +cargo check --manifest-path "$work/Cargo.toml" +if DUCKDB_DOWNLOAD_LIB=1 cargo check --manifest-path "$work/Cargo.toml" >"$work/rejected" 2>&1; then + echo 'download without its feature was accepted' >&2; exit 1 +fi +grep -q 'requires the libduckdb-sys/download feature' "$work/rejected" +# Explicit feature must compile the original downloader. Keep network fetching +# disabled here; the download itself is a separately reported integration check. +cargo check --manifest-path "$work/Cargo.toml" --features download +cargo tree --manifest-path "$work/Cargo.toml" --features bundled --edges build,normal >"$work/bundled-tree" +if grep -Eq 'reqwest|rustls|zip v' "$work/bundled-tree"; then + echo 'bundled native dependency still includes the downloader' >&2; exit 1 +fi +# A fresh CI checkout uses stable mtimes, and refuses to normalize dirty source +# or an existing target. These fixtures do not establish Cargo artifact freshness. +fixture="$work/fixture" +mkdir -p "$fixture/.github/scripts" "$fixture/rust_hft/native/libduckdb-sys" +cp "$root/.github/scripts/prepare-research-native-source.sh" "$fixture/.github/scripts/" +printf 'native fixture' >"$fixture/rust_hft/native/libduckdb-sys/build.rs" +git -C "$fixture" init -q +git -C "$fixture" add . +git -C "$fixture" -c user.name=fixture -c user.email=fixture@example.invalid commit -qm fixture +GITHUB_ACTIONS=true bash "$fixture/.github/scripts/prepare-research-native-source.sh" +ruby -e 'abort "unstable mtime" unless File.mtime(ARGV[0]).to_i == 946684800' "$fixture/rust_hft/native/libduckdb-sys/build.rs" +mkdir -p "$fixture/rust_hft/target/research-core" +if GITHUB_ACTIONS=true bash "$fixture/.github/scripts/prepare-research-native-source.sh"; then exit 1; fi +rmdir "$fixture/rust_hft/target/research-core" +printf 'dirty' >>"$fixture/rust_hft/native/libduckdb-sys/build.rs" +if GITHUB_ACTIONS=true bash "$fixture/.github/scripts/prepare-research-native-source.sh"; then exit 1; fi +printf 'PASS: system and downloader compile, missing feature rejects, bundled graph excludes download dependencies, timestamp guards reject stale/dirty preparation\n' diff --git a/.github/scripts/test-loop-nextest.sh b/.github/scripts/test-loop-nextest.sh index 09719e019..65216dc85 100755 --- a/.github/scripts/test-loop-nextest.sh +++ b/.github/scripts/test-loop-nextest.sh @@ -247,16 +247,18 @@ abort 'loop step still runs cargo test' if loop.fetch('run').include?('cargo-sco abort 'loop step does not wait for shards' unless loop.fetch('run').include?('loop-nextest-wait.sh') archive = jobs.fetch('rust_loop_nextest_archive') abort 'archive must depend only on scope' unless archive.fetch('needs') == 'scope' -cache = archive.fetch('steps').find { |step| step['uses'].to_s.include?('Swatinem/rust-cache@') } -abort 'archive cache key drifted' unless cache && cache.dig('with', 'key') == 'loop-nextest-research-owner-v2-${{ steps.cache-info.outputs.rust }}' -cache_pairs = cache.dig('with', 'workspaces').lines.map(&:strip).reject(&:empty?).map { |line| line.split(/\s*->\s*/) } -abort 'archive cleanup must use only the research owner' unless cache_pairs.length == 1 && cache_pairs[0][0] == 'rust_hft/research-core' -cache_source, cache_target = cache_pairs.fetch(0) -abort 'archive cache target is missing' unless cache_target -target = Pathname.new(cache_source).join(cache_target).cleanpath.to_s -build_target = archive.fetch('env').fetch('CARGO_TARGET_DIR').delete_prefix('${{ github.workspace }}/') -abort 'archive cache does not cover its Cargo target' unless target == build_target -abort 'archive cache save drifted' unless cache.dig('with', 'save-if') == "${{ github.ref == 'refs/heads/main' }}" +archive_steps = archive.fetch('steps') +cache = archive_steps.find { |step| step['uses'].to_s.start_with?('actions/cache/restore@') } +save = archive_steps.find { |step| step['uses'].to_s.start_with?('actions/cache/save@') } +abort 'archive must retain its path native dependency' unless cache && save && archive.dig('env', 'MONDAY_CARGO_TARGET_LAYOUT') == 'owning-workspace-v1' +abort 'archive cache key drifted' unless cache.dig('with', 'key') == 'loop-nextest-${{ steps.cache-info.outputs.key }}' +abort 'archive restore/save identities differ' unless %w[key path].all? { |key| cache.dig('with',key) == save.dig('with',key) } +abort 'archive compatible restore missing' unless cache.dig('with','restore-keys') == 'loop-nextest-${{ steps.cache-info.outputs.compat_prefix }}' +cleanup = archive_steps.index { |step| step['run'].to_s.include?('ci-owner-cache.sh" cleanup') } +abort 'archive cache must clean local application bytes before save' unless cleanup && cleanup < archive_steps.index(save) +abort 'archive cache is not main/success bound' unless save.fetch('if').include?("github.ref == 'refs/heads/main' && success()") +prepare = archive_steps.index { |step| step['run'].to_s.include?('prepare-research-native-source.sh') } +abort 'archive native source preparation must precede restore' unless prepare && prepare < archive_steps.index(cache) shard = jobs.fetch('rust_loop_nextest_shard') abort 'shard must depend only on the archive' unless shard.fetch('needs') == 'rust_loop_nextest_archive' abort 'shard fail-fast must stay off' unless shard.dig('strategy', 'fail-fast') == false diff --git a/.github/scripts/test-research-cache-layout.sh b/.github/scripts/test-research-cache-layout.sh index 866935b90..9c1aed346 100644 --- a/.github/scripts/test-research-cache-layout.sh +++ b/.github/scripts/test-research-cache-layout.sh @@ -23,6 +23,13 @@ while IFS= read -r manifest; do cp "$root/rust_hft/$manifest" "$compat_repo/rust_hft/$manifest" printf 'fixture lock\n' >"$compat_repo/rust_hft/${manifest%Cargo.toml}Cargo.lock" done < <(jq -r '.workspaces[].manifest' "$root/rust_hft/workspaces.json") +# Use the real compiler recipe input path, not a fabricated recipe digest. +for script in build-research-release.sh build-research-recipes.sh capture-research-build-inputs.sh research-release-products.sh \ + research-release-source-sha.sh research-workspace-locks.sh verify-research-runtime-abi.sh \ + verify-research-runner-binaries.sh research-native-dependency-inputs.rb compile-research-recipe.sh prepare-research-native-source.sh; do + cp "$root/.github/scripts/$script" "$compat_repo/.github/scripts/" +done +cp "$root/.github/scripts/research-image-release-artifact.sh" "$compat_repo/.github/scripts/" printf '[features]\ndefault=[]\n' >"$compat_repo/rust_hft/alpha-harness/app/Cargo.toml" git -C "$compat_repo" init -q git -C "$compat_repo" add rust_hft @@ -40,12 +47,62 @@ fixture_inputs() { digest=$(printf 'compiler/native fixture\n' | sha256sum | awk '{print $1}') jq -Sn --arg digest "$digest" --arg profiles "$(sha256sum "$work/profiles" | awk '{print $1}')" \ --argjson owners "$profiles" \ + --arg recipe "$(bash "$compat_helper" recipe-inputs | sha256sum | awk '{print $1}')" \ --argjson locks "$(bash "$root/.github/scripts/research-workspace-locks.sh" "$compat_repo/rust_hft")" \ - --argjson recipes "$(bash "$root/.github/scripts/research-release-products.sh" recipes "${1:-cex-runner,controller}" | jq -s .)" \ + --argjson recipes "$(bash "$compat_repo/.github/scripts/research-release-products.sh" recipes "${1:-cex-runner,controller}" | jq -s .)" \ '{schema:"monday.compilation-inputs.v3",target:"x86_64-unknown-linux-gnu",profile:"release", - compiler:$digest,native:$digest,flags:$digest,profiles:$profiles,recipe:$digest, + compiler:$digest,native:$digest,flags:$digest,profiles:$profiles,recipe:$recipe, builder_image:("rust:fixture@sha256:"+$digest),recipes:$recipes,workspace_profiles:$owners,locks:$locks}' } +# Metadata fixtures exercise identity derivation without compiling DuckDB. +original_path=$PATH +mkdir -p "$work/native-bin" +cat >"$work/native-bin/cargo" <<'NATIVE_CARGO' +#!/usr/bin/env bash +set -euo pipefail +[[ $1 == metadata ]] +if [[ -n ${MONDAY_NATIVE_CALLS:-} ]]; then printf '%s\n' "$*" >>"$MONDAY_NATIVE_CALLS"; fi +cat "$MONDAY_NATIVE_FIXTURE" +NATIVE_CARGO +chmod +x "$work/native-bin/cargo" +export MONDAY_NATIVE_FIXTURE="$work/native-metadata.json" +export PATH="$work/native-bin:$PATH" +cat >"$MONDAY_NATIVE_FIXTURE" <<'NATIVE_JSON' +{"packages":[{"id":"native","name":"libduckdb-sys","version":"1.10504.0","source":"registry+https://github.com/rust-lang/crates.io-index"},{"id":"cc","name":"cc","version":"1.2.0","source":"registry+https://github.com/rust-lang/crates.io-index"}],"resolve":{"nodes":[{"id":"native","features":["bundled"],"dependencies":["cc"]},{"id":"cc","features":[],"dependencies":[]}]}} +NATIVE_JSON +cp "$MONDAY_NATIVE_FIXTURE" "$work/native-original.json" +native_fixture_locks() { + ruby -rjson - "$MONDAY_NATIVE_FIXTURE" "$compat_repo/rust_hft" <<'NATIVE_LOCKS' +metadata, root = ARGV +lock = "version = 3\n" + JSON.parse(File.read(metadata)).fetch('packages').map do |package| + "[[package]]\nname=#{package.fetch('name').to_json}\nversion=#{package.fetch('version').to_json}\nsource=#{package.fetch('source').to_json}\nchecksum=\"#{'1' * 64}\"\n" +end.join +Dir.glob("#{root}/**/Cargo.lock").each { |path| File.write(path, lock) } +NATIVE_LOCKS +} +native_fixture_locks +# Repeated recipes share resolution only when the full Cargo command is identical. +export MONDAY_NATIVE_CALLS="$work/native-calls" +bash "$compat_repo/.github/scripts/research-release-products.sh" recipes all | jq -s . >"$work/native-recipes.json" +ruby "$compat_repo/.github/scripts/research-native-dependency-inputs.rb" "$compat_repo" "$work/native-recipes.json" >"$work/native-deduplicated.json" +[[ $(wc -l <"$MONDAY_NATIVE_CALLS") -eq 4 ]] +# Running each recipe separately is the prior behavior and identity oracle. +: >"$work/native-separate.jsonl" +while IFS= read -r recipe; do + printf '[%s]\n' "$recipe" >"$work/native-single.json" + ruby "$compat_repo/.github/scripts/research-native-dependency-inputs.rb" "$compat_repo" "$work/native-single.json" >>"$work/native-separate.jsonl" +done < <(jq -c '.[]' "$work/native-recipes.json") +jq -sS '{schema:.[0].schema,closures:([.[].closures[]]|unique)}' "$work/native-separate.jsonl" >"$work/native-expected.json" +jq -S '.closures |= sort' "$work/native-deduplicated.json" >"$work/native-actual.json" +cmp "$work/native-expected.json" "$work/native-actual.json" +# Different package-qualified feature arguments must not share resolution. +jq '[.[0], (.[0] + {features:"extra"}), (.[1] + {features:"extra"}), .[0]]' "$work/native-recipes.json" >"$work/native-features.json" +: >"$MONDAY_NATIVE_CALLS" +ruby "$compat_repo/.github/scripts/research-native-dependency-inputs.rb" "$compat_repo" "$work/native-features.json" >/dev/null +[[ $(wc -l <"$MONDAY_NATIVE_CALLS") -eq 3 ]] +unset MONDAY_NATIVE_CALLS +printf 'PASS: native metadata resolves four distinct commands instead of eight recipes; identity matches separate resolution\n' +cp "$compat_repo/rust_hft/research-core/Cargo.lock" "$work/original-fixture-lock" fixture_inputs >"$work/full.json" cp "$work/full.json" "$work/full-before.json" bash "$compat_helper" compatibility-inputs "$work/full.json" >"$work/compat.json" @@ -59,19 +116,103 @@ check_identity_change() { actual=$(sha256sum "$work/changed-compat.json" | awk '{print $1}') if [[ $expected == reuse ]]; then [[ $actual == "$compat_sha" ]]; else [[ $actual != "$compat_sha" ]]; fi } +# A valid catalog edit changes exact provenance, not dependency compatibility. +# Both feature sets build the same operator binary; only the recipe identity differs. +catalog="$compat_repo/.github/scripts/research-release-products.json" +cp "$catalog" "$work/catalog-original" +release="$work/catalog-release" +mkdir -p "$release/research-bin" +while IFS= read -r binary; do + printf 'catalog fixture executable: %s\n' "$binary" >"$release/research-bin/$binary" + chmod 0755 "$release/research-bin/$binary" +done < <(bash "$compat_repo/.github/scripts/research-release-products.sh" binaries cex-runner,controller) +artifact="$compat_repo/.github/scripts/research-image-release-artifact.sh" +source_sha=1111111111111111111111111111111111111111 +MONDAY_BUILD_INPUTS_FILE="$work/full.json" bash "$artifact" create "$release" "$source_sha" 42 "$compat_repo/rust_hft" 1 7 cex-runner,controller +bash "$artifact" verify "$release" "$source_sha" 42 "$compat_repo/rust_hft" 1 7 cex-runner,controller +bash "$compat_helper" recipe-inputs >"$work/compiler-recipe" +jq '(.recipes[] | select(.package=="alpha-harness" and .binaries==["alpha-harness"]) | .features)="operator"' \ + "$work/catalog-original" >"$catalog" +fixture_inputs >"$work/changed.json" +check_identity_change reuse +bash "$compat_helper" recipe-inputs >"$work/catalog-recipe" +cmp "$work/compiler-recipe" "$work/catalog-recipe" +if bash "$artifact" verify "$release" "$source_sha" 42 "$compat_repo/rust_hft" 1 7 cex-runner,controller >"$work/rejection" 2>&1; then + echo 'old catalog recipes were accepted as current provenance' >&2; exit 1 +fi +# The same executable fixture with current provenance is valid. This isolates +# the rejection above to the recipe change, rather than an unrelated fixture error. +rm "$release/research-image-release.json" +MONDAY_BUILD_INPUTS_FILE="$work/changed.json" bash "$artifact" create "$release" "$source_sha" 42 "$compat_repo/rust_hft" 1 7 cex-runner,controller +bash "$artifact" verify "$release" "$source_sha" 42 "$compat_repo/rust_hft" 1 7 cex-runner,controller +cp "$work/catalog-original" "$catalog" +printf '\n# changed compiler orchestration\n' >>"$compat_repo/.github/scripts/build-research-recipes.sh" +fixture_inputs >"$work/changed.json" +check_identity_change reuse +cp "$root/.github/scripts/build-research-recipes.sh" "$compat_repo/.github/scripts/" +for script in build-research-release.sh verify-research-runtime-abi.sh verify-research-runner-binaries.sh; do + printf '\n# packaging or validation text change\n' >>"$compat_repo/.github/scripts/$script" + fixture_inputs >"$work/changed.json" + check_identity_change reuse + cp "$root/.github/scripts/$script" "$compat_repo/.github/scripts/" +done +for script in compile-research-recipe.sh research-cache-layout.sh prepare-research-native-source.sh; do + printf '\n# changed actual compile or cleanup contract\n' >>"$compat_repo/.github/scripts/$script" + fixture_inputs >"$work/changed.json" + check_identity_change isolate + cp "$root/.github/scripts/$script" "$compat_repo/.github/scripts/" +done +printf 'PASS: valid catalog edit preserves dependency identity, changes exact inputs and rejects old artifact recipes\n' +# Cargo-resolved native version/source/features and build dependencies must isolate. +for update in '.packages[0].version="1.10505.0"' '.packages[0].source="git+https://example.invalid/duckdb#1234"' \ + '.resolve.nodes[0].features=["bundled","json"]' '.packages[1].version="1.3.0"'; do + jq "$update" "$work/native-original.json" >"$MONDAY_NATIVE_FIXTURE" + native_fixture_locks + bash "$compat_helper" compatibility-inputs "$work/full.json" >"$work/native-changed.json" + [[ $(sha256sum "$work/native-changed.json" | awk '{print $1}') != "$compat_sha" ]] +done +cp "$work/native-original.json" "$MONDAY_NATIVE_FIXTURE" +native_fixture_locks +jq '.packages += [{id:"business",name:"unrelated-business",version:"2.0.0",source:null}]' \ + "$work/native-original.json" >"$MONDAY_NATIVE_FIXTURE" +bash "$compat_helper" compatibility-inputs "$work/full.json" >"$work/native-unrelated.json" +cmp "$work/compat.json" "$work/native-unrelated.json" +cp "$work/native-original.json" "$MONDAY_NATIVE_FIXTURE" +ruby - "$compat_repo/rust_hft/research-core/Cargo.lock" <<'NATIVE_CHECKSUM' +path = ARGV.fetch(0) +File.write(path, File.read(path).sub('1' * 64, '2' * 64)) +NATIVE_CHECKSUM +bash "$compat_helper" compatibility-inputs "$work/full.json" >"$work/native-checksum.json" +[[ $(sha256sum "$work/native-checksum.json" | awk '{print $1}') != "$compat_sha" ]] +native_fixture_locks +# A path patch binds native archive and build script bytes, not checkout mtimes. +mkdir -p "$compat_repo/rust_hft/native/libduckdb-sys" +printf 'native archive fixture' >"$compat_repo/rust_hft/native/libduckdb-sys/duckdb.tar.gz" +printf 'native build fixture' >"$compat_repo/rust_hft/native/libduckdb-sys/build.rs" +jq --arg path "$compat_repo/rust_hft/native/libduckdb-sys/Cargo.toml" \ + '.packages[0].source=null | .packages[0].manifest_path=$path' \ + "$work/native-original.json" >"$MONDAY_NATIVE_FIXTURE" +bash "$compat_helper" compatibility-inputs "$work/full.json" >"$work/native-local-before.json" +for file in duckdb.tar.gz build.rs; do + printf 'changed bytes' >>"$compat_repo/rust_hft/native/libduckdb-sys/$file" + bash "$compat_helper" compatibility-inputs "$work/full.json" >"$work/native-local-after.json" + ! cmp -s "$work/native-local-before.json" "$work/native-local-after.json" + cp "$work/native-local-after.json" "$work/native-local-before.json" +done +cp "$work/native-original.json" "$MONDAY_NATIVE_FIXTURE" # Local feature and lock changes remain in complete provenance, not the restore prefix. cp "$compat_repo/rust_hft/alpha-harness/app/Cargo.toml" "$work/leaf-before" printf '[features]\ndefault=["scientific"]\n' >"$compat_repo/rust_hft/alpha-harness/app/Cargo.toml" fixture_inputs >"$work/changed.json" check_identity_change reuse cp "$work/leaf-before" "$compat_repo/rust_hft/alpha-harness/app/Cargo.toml" -printf 'new fixture lock\n' >"$compat_repo/rust_hft/research-core/Cargo.lock" +printf '\n# unrelated lock metadata\n' >>"$compat_repo/rust_hft/research-core/Cargo.lock" fixture_inputs >"$work/changed.json" check_identity_change reuse -printf 'fixture lock\n' >"$compat_repo/rust_hft/research-core/Cargo.lock" +cp "$work/original-fixture-lock" "$compat_repo/rust_hft/research-core/Cargo.lock" changed_digest=$(printf 'changed environment\n' | sha256sum | awk '{print $1}') -for field in compiler native flags recipe; do +for field in compiler native flags; do jq -S --arg field "$field" --arg digest "$changed_digest" '.[$field]=$digest' "$work/full.json" >"$work/changed.json" check_identity_change isolate done @@ -226,6 +367,7 @@ end abort 'producer cache namespaces drifted' unless restores.uniq.length == 1 RUBY printf 'PASS: dependency compatibility identity, provenance preservation and fail-closed input contracts\n' +export PATH="$original_path" [[ ${1:-} != --compatibility-key ]] || exit 0 bash "$root/.github/scripts/research-release-products.sh" recipes all | jq -s '{recipes:.}' >"$work/inputs" @@ -254,7 +396,7 @@ jq -n --arg root "$FIXTURE/repo" --argjson extra "$extra" '{packages:([ {name:"hidden-local",manifest_path:($root+"/vendor/hidden-local/Cargo.toml"),targets:[{name:"hidden_local"}]}, {name:"outside-local",source:null,manifest_path:"/external/local/Cargo.toml",targets:[{name:"outside_local"}]}, {name:"data-encoding",source:"registry+https://example.invalid",manifest_path:"/external/registry/data-encoding/Cargo.toml",targets:[{name:"data_encoding"}]}, - {name:"libduckdb-sys",source:"registry+https://example.invalid",manifest_path:"/external/registry/libduckdb-sys/Cargo.toml",targets:[{name:"libduckdb_sys"}]}, + {name:"libduckdb-sys",source:null,manifest_path:($root+"/rust_hft/native/libduckdb-sys/Cargo.toml"),targets:[{name:"libduckdb_sys"}]}, {name:"local-code",source:"registry+https://example.invalid",manifest_path:"/external/registry/local-code/Cargo.toml",targets:[{name:"local_code"}]} ] else [] end)}' MOCK diff --git a/.github/scripts/test-research-publication-budget.sh b/.github/scripts/test-research-publication-budget.sh index adbc6cbb1..47041d9e8 100644 --- a/.github/scripts/test-research-publication-budget.sh +++ b/.github/scripts/test-research-publication-budget.sh @@ -10,9 +10,9 @@ products=cex-runner,controller,prediction-runner bash "$script_dir/research-publication-budget.sh" estimate "$source_sha" "$products" 744 "$work/estimate.json" jq -e ' .basis=="catalog-static-upper-bound" and .source_archive.bytes>0 and - [.allocations[].executable_count]==[9,1,5] and [.allocations[].build_count]==[6,1,3] and - [.allocations[].native_requests]==[90,21,51] and - .total.object_count==48 and .total.oss_requests==150 and .total.native_requests==162 and + [.allocations[].executable_count]==[8,1,5] and [.allocations[].build_count]==[5,1,3] and + [.allocations[].native_requests]==[78,21,51] and + .total.object_count==44 and .total.oss_requests==138 and .total.native_requests==150 and .total.request_body_bytes==(.allocations|map(.request_body_bytes)|add) and .total.response_body_bytes==(.allocations|map(.response_body_bytes)|add) and .pricing.estimated_micro_cny==(.pricing.request_micro_cny+.pricing.egress_micro_cny+.pricing.storage_micro_cny) and @@ -46,7 +46,7 @@ run_admit() { run_admit "$work/policy.json" jq -e '.admission.single_run and .admission.publisher_run_id==456 and .admission.publisher_run_number==123' \ "$work/admission.json" >/dev/null -jq -e '.schema=="monday.oss-publication-budget.v1" and .limits.requests==162 and +jq -e '.schema=="monday.oss-publication-budget.v1" and .limits.requests==150 and .publisher_run_id==456 and .publisher_run_attempt==1 and .expires_at_ms==1791684000000 and .publication_namespaces==["research/builds/","research/sources/"] and .limits.requests==(.allocations|map(.limits.requests)|add) and @@ -150,7 +150,7 @@ jq --slurpfile estimate "$work/pair-estimate.json" ' .max_request_body_bytes=$estimate[0].total.request_body_bytes | .max_response_body_bytes=$estimate[0].total.response_body_bytes | .max_new_storage_bytes=$estimate[0].total.new_storage_bytes' "$work/operations.json" >"$work/pair-policy.json" -jq -e '.total.oss_requests==103' "$work/pair-estimate.json" >/dev/null +jq -e '.total.oss_requests==91' "$work/pair-estimate.json" >/dev/null for order in 'cex-runner controller' 'controller cex-runner'; do run=500 for product in $order; do @@ -182,7 +182,7 @@ done # Both single-product estimates individually fit these underfunded source caps; # neither may spend until the complete fixed set fits. A failed sibling does not # return its reserved share to the other product. -for expression in '.max_oss_requests=102' '.max_estimated_micro_cny-=1' \ +for expression in '.max_oss_requests=90' '.max_estimated_micro_cny-=1' \ '.max_request_body_bytes-=1' '.max_response_body_bytes-=1' '.max_new_storage_bytes-=1'; do jq "$expression" "$work/pair-policy.json" >"$work/undersized-pair-policy.json" for product in cex-runner controller; do diff --git a/.github/scripts/test-research-release-products.sh b/.github/scripts/test-research-release-products.sh index 05b042811..725fb747d 100755 --- a/.github/scripts/test-research-release-products.sh +++ b/.github/scripts/test-research-release-products.sh @@ -13,11 +13,11 @@ products="$root/.github/scripts/research-release-products.sh" bash "$products" recipes prediction-runner >"$work/prediction-recipes" jq -s -e 'length==3 and all(.[]; .manifest=="prediction-markets/Cargo.toml") and any(.[]; .package=="ploy-research" and (.binaries|length)==3) and any(.[]; .package=="hft-prediction-research-worker" and .binaries==["monday-prediction-worker"]) and any(.[]; .package=="hft-prediction-research-operator" and .binaries==["monday-prediction-operator"])' "$work/prediction-recipes" >/dev/null bash "$products" recipes cex-runner >"$work/cex-recipes" -jq -s -e 'length==6 and any(.[]; .package=="hft-cex-data-preparation" and .manifest=="data-pipelines/Cargo.toml" and .binaries==["monday-cex-prepare"]) and any(.[]; .package=="hft-cex-research-worker" and .binaries==["monday-cex-worker"]) and all(.[]; .package!="ploy-research" and .manifest!="prediction-markets/Cargo.toml")' "$work/cex-recipes" >/dev/null -jq -s -e 'any(.[]; .package=="alpha-harness" and .features=="offline-diagnostic" and .binaries==["monday-offline-baseline"]) and any(.[]; .package=="alpha-harness" and .features=="" and .binaries==["alpha-harness"])' "$work/cex-recipes" >/dev/null -[[ $(bash "$products" binaries cex-runner | wc -l) -eq 9 ]] -[[ $(bash "$products" binaries all | wc -l) -eq 14 ]] -[[ $(bash "$products" binaries cex-runner,controller | wc -l) -eq 9 ]] +jq -s -e 'length==5 and any(.[]; .package=="hft-cex-data-preparation" and .manifest=="data-pipelines/Cargo.toml" and .binaries==["monday-cex-prepare"]) and any(.[]; .package=="hft-cex-research-worker" and .binaries==["monday-cex-worker"]) and all(.[]; .package!="ploy-research" and .manifest!="prediction-markets/Cargo.toml")' "$work/cex-recipes" >/dev/null +jq -s -e 'all(.[]; (.binaries|index("monday-offline-baseline"))==null) and any(.[]; .package=="alpha-harness" and .features=="" and .binaries==["alpha-harness"])' "$work/cex-recipes" >/dev/null +[[ $(bash "$products" binaries cex-runner | wc -l) -eq 8 ]] +[[ $(bash "$products" binaries all | wc -l) -eq 13 ]] +[[ $(bash "$products" binaries cex-runner,controller | wc -l) -eq 8 ]] [[ $(bash "$products" merge controller prediction-runner) == controller,prediction-runner ]] for invalid in runner paired '' 'cex-runner,cex-runner' 'prediction-runner,unknown' 'all,controller'; do if bash "$products" normalize "$invalid" >"$work/rejection" 2>&1; then @@ -33,6 +33,10 @@ catalog=JSON.parse(File.read(File.join(root,'.github/scripts/research-release-pr prebuilt=text.split(/^FROM runtime-base AS prebuilt\s*$/,2).fetch(1).split(/^FROM /,2).first copied=prebuilt.scan(/^COPY --chmod=0755 research-bin\/(\S+) \/usr\/local\/bin\/(\S+)$/) abort "unexpected prebuilt COPY: #{product}" unless copied.all? { |source,target| source==target } && copied.map(&:first).sort==catalog.fetch(product).sort + builder=text.split(/^FROM .* AS runtime-base\s*$/,2).first + installed=builder.scan(/install -D -m 0755 (\S+) \/out\/bin\/(\S+)/) + target=product=='cex-runner' ? '/work/target/release' : '/work/target/prediction-markets/release' + abort "unexpected source install: #{product}" unless installed.all? { |source,binary| source=="#{target}/#{binary}" } && installed.map(&:last).sort==catalog.fetch(product).sort foreign=catalog.fetch(product=='cex-runner' ? 'prediction-runner' : 'cex-runner') abort "source builder imports another domain: #{product}" if foreign.any? { |binary| text.include?(binary) } end @@ -88,7 +92,7 @@ bash "$root/.github/scripts/test-research-product-image.sh" # binaries. Only the target recorded by compiler inputs may reach the archive. fixture="$work/producer" mkdir -p "$fixture/.github/scripts" "$fixture/rust_hft" "$fixture/bin" "$fixture/output" -for script in research-cache-layout.sh build-research-release.sh build-research-recipes.sh research-release-source-sha.sh research-release-products.sh research-release-products.json research-workspace-locks.sh research-image-release-artifact.sh verify-research-runner-binaries.sh research-release-bundle.rb; do +for script in research-cache-layout.sh build-research-release.sh build-research-recipes.sh compile-research-recipe.sh research-release-source-sha.sh research-release-products.sh research-release-products.json research-workspace-locks.sh research-image-release-artifact.sh verify-research-runner-binaries.sh research-release-bundle.rb; do cp "$root/.github/scripts/$script" "$fixture/.github/scripts/$script" done cp "$root/rust_hft/workspaces.json" "$fixture/rust_hft/workspaces.json" @@ -143,6 +147,9 @@ if [[ $1 == metadata ]]; then jq -n --arg root "$PWD" '{packages:[{name:"alpha-harness",manifest_path:($root+"/alpha-harness/Cargo.toml"),targets:[{name:"alpha_harness"}]}]}' exit fi +[[ ${CARGO_LOG:-} == cargo::core::compiler::fingerprint=info ]] +[[ ${CARGO_TERM_COLOR:-} == never ]] +printf ' INFO cargo::core::compiler::fingerprint: fingerprint error for libduckdb-sys: missing fixture fingerprint\n' >&2 printf 'build\n' >>"${MONDAY_TEST_CARGO_BUILD_LOG:?}" [[ ${FAIL_RECIPE:-0} != 1 ]] || exit 37 target='' manifest='' package='' features='' jobs='' binaries=() @@ -171,6 +178,17 @@ File.open(file, 'a') { |out| out.flock(File::LOCK_EX); out.puts(JSON.generate(re RUBY } event start + if [[ ${MONDAY_TEST_CARGO_MODE:-} == reclaim && $package == alpha-harness ]]; then + # Hold the slow owner until the supervisor observes both fast owners exit. + # Its current invocation keeps two jobs; only its next recipe may use four. + released=0 + for ((wait_count=0; wait_count<200; wait_count++)); do + leases=("$RUNNER_TEMP"/research-recipe-jobs.*/research-core.jobs) + if [[ ${#leases[@]} == 1 && -f ${leases[0]} && $(cat "${leases[0]}") == 4 ]]; then released=1; break; fi + sleep 0.05 + done + (( released == 1 )) || { echo 'final owner never reclaimed idle CPUs' >&2; exit 95; } + fi if [[ ${MONDAY_TEST_CARGO_MODE:-} == fail-and-cancel || ${MONDAY_TEST_CARGO_MODE:-} == interrupted ]]; then if [[ $package == hft-collector ]]; then ruby -e 'Signal.trap("TERM") {}; File.write(ARGV[0], Process.pid.to_s); sleep 30' "${MONDAY_TEST_COMPILER_PID:?}" & @@ -218,6 +236,7 @@ jq --argjson count "$(bash "$products" recipes controller | jq -s length)" -es ' and ([.[].phase]|unique)==["after-cache-lookup","dependency-warm-parallel","dependency-warm-serial","warm-local"] and ([.[].compilation_inputs_sha256]|unique|length)==1 and ([.[].build_inputs]|unique|length)==1 + and all(.[]; (.native_fingerprint_diagnostics|length)==1 and (.native_fingerprint_diagnostics[0]|contains("libduckdb-sys"))) and all(.[]; .build_inputs.profile=="release" and .build_inputs.target=="x86_64-unknown-linux-gnu" and .phase_seconds>=0) and all(.[]|select(.phase=="dependency-warm-serial"); .scheduler.requested_owner_concurrency==1 and .scheduler.owner_concurrency==1 and .scheduler.cargo_jobs==8) and all(.[]|select(.phase=="dependency-warm-parallel"); .scheduler.requested_owner_concurrency==2 and .scheduler.owner_concurrency==1 and .scheduler.cargo_jobs==8) @@ -283,9 +302,41 @@ events.sort_by { |event| event.fetch('at') }.each do |event| end abort 'independent owners never overlapped' unless maximum == 2 && active.empty? RUBY -jq --argjson count "$(bash "$products" recipes all | jq -s length)" -es 'length==$count and all(.[]; .scheduler.owner_concurrency==2 and .scheduler.cargo_jobs==2)' \ +jq --argjson count "$(bash "$products" recipes all | jq -s length)" -es 'length==$count and all(.[]; .scheduler.owner_concurrency==2 and (.scheduler.cargo_jobs==2 or .scheduler.cargo_jobs==4))' \ "$fixture/parallel-output/research-recipe-probe/timings.jsonl" >/dev/null +# Reclaim only after both other owners exit, without changing an in-flight build. +rm -f "$MONDAY_TEST_CARGO_EVENTS" +mkdir -p "$fixture/reclaim-output" +MONDAY_TEST_CARGO_MODE=reclaim PATH="$fixture/bin:$PATH" RUNNER_TEMP="$fixture/reclaim-output" MONDAY_SOURCE_REVISION="$sha" \ + MONDAY_BUILD_INPUTS_FILE="$fixture/all-inputs.json" CARGO_BUILD_JOBS=4 \ + bash "$fixture/.github/scripts/build-research-recipes.sh" all after-cache-lookup >"$work/reclaim.log" +ruby -rjson - "$MONDAY_TEST_CARGO_EVENTS" <<'RUBY' +events = File.readlines(ARGV[0]).map { |line| JSON.parse(line) }.sort_by { |event| event.fetch('at') } +active = {} +reclaimed = false +events.each do |event| + owner = event.fetch('manifest') + if event.fetch('event') == 'start' + abort 'overlapping owner target writes' if active.key?(owner) + if event.fetch('package') == 'alpha-harness' + abort 'in-flight budget changed' unless event.fetch('jobs') == 2 + elsif event.fetch('package') == 'hft-cex-research-worker' + abort 'idle CPUs were not reclaimed' unless event.fetch('jobs') == 4 && active.empty? + reclaimed = true + end + active[owner] = event.fetch('jobs') + else + abort 'completion without owner' unless active.delete(owner) + end + abort 'CPU or owner budget exceeded' unless active.length <= 2 && active.values.sum <= 4 +end +abort 'reclaim evidence incomplete' unless reclaimed && active.empty? +RUBY +jq -es 'any(.[]; .recipe.package=="hft-cex-research-worker" and .scheduler.cargo_jobs==4)' \ + "$fixture/reclaim-output/research-recipe-probe/timings.jsonl" >/dev/null +[[ -z $(find "$fixture/reclaim-output" -name 'research-recipe-jobs.*' -print) ]] + # Low available memory reduces overlap. Explicit serial mode uses the same recipes. printf 'MemAvailable: 4194304 kB\n' >"$MONDAY_TEST_MEMINFO_FILE" for mode in memory serial; do @@ -306,6 +357,7 @@ MONDAY_TEST_CARGO_MODE=fail-and-cancel PATH="$fixture/bin:$PATH" RUNNER_TEMP="$f GITHUB_REPOSITORY=fixture/monday GITHUB_RUN_ID=42 MONDAY_BUILD_INPUTS_FILE="$fixture/all-inputs.json" \ bash "$fixture/.github/scripts/build-research-release.sh" all >"$work/cancel.log" 2>&1 || code=$? [[ $code == 37 && ! -e $fixture/cancel-output/research-image-release.tar ]] +[[ -z $(find "$fixture/cancel-output" -name 'research-recipe-jobs.*' -print) ]] for pid_file in "$MONDAY_TEST_COMPILER_PID" "$MONDAY_TEST_FAILED_COMPILER_PID"; do compiler_pid=$(cat "$pid_file") # PID 1 may retain an exited orphan as a zombie; it cannot execute anymore. @@ -356,6 +408,7 @@ ensure end RUBY [[ ! -e $fixture/interrupted-output/research-image-release.tar ]] +[[ -z $(find "$fixture/interrupted-output" -name 'research-recipe-jobs.*' -print) ]] # Invalid resource requests fail before starting a compiler. for request in cpu concurrency; do diff --git a/.github/scripts/test-rust-docker-workspaces.rb b/.github/scripts/test-rust-docker-workspaces.rb index f21ab055b..2be89d199 100644 --- a/.github/scripts/test-rust-docker-workspaces.rb +++ b/.github/scripts/test-rust-docker-workspaces.rb @@ -30,6 +30,12 @@ def tracked(pattern) block = block.split(/^\[/, 2)[0] name = block[/^name\s*=\s*"([^"]+)"/, 1] next unless name + # This checksum-documented native dependency is not an application owner. + if relative == 'rust_hft/native/libduckdb-sys/Cargo.toml' + abort 'unexpected native patch package' unless name == 'libduckdb-sys' && + block.match?(/^version\s*=\s*"1\.10504\.0"/) && text.match?(/^\[workspace\]/) + next + end declared = block[/^workspace\s*=\s*"([^"]+)"/, 1] owner = if declared File.expand_path(File.join(declared, 'Cargo.toml'), File.dirname(file)) diff --git a/.github/scripts/test-select-rust-ci-scope.sh b/.github/scripts/test-select-rust-ci-scope.sh index b7d90f495..b72e1db3f 100755 --- a/.github/scripts/test-select-rust-ci-scope.sh +++ b/.github/scripts/test-select-rust-ci-scope.sh @@ -708,6 +708,12 @@ manual_scope=$(run_case production-manual workflow_dispatch dev-input.txt "$prod assert_flag "$manual_scope" research_product cex-runner,controller,prediction-runner assert_flag "$manual_scope" production_trading_image true assert_flag "$manual_scope" production_collector_image true +for helper in compile-research-recipe.sh research-native-dependency-inputs.rb prepare-research-native-source.sh test-duckdb-build-modes.sh; do + printf '%s\n' ".github/scripts/$helper" >"$tmp_dir/native-compile-helper.txt" + native_output=$(run_case native-compile-helper pull_request native-compile-helper.txt) + assert_jobs "$native_output" 'ploy/commit-hygiene,ploy/research-image-binaries,ploy/research-image-smoke,ploy/safety-scans,ci/ci-contracts,ploy/workflow-lint' +done + if [[ ${1:-} == --production-scope ]]; then printf 'production dependency scope contracts passed\n' exit 0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e4d51a74..9a23c7dd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -209,6 +209,10 @@ jobs: uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable with: toolchain: 1.98.1 + - name: Verify DuckDB build modes and source freshness guards + env: + MONDAY_NATIVE_TEST_TARGET: ${{ runner.temp }}/native-mode-target + run: bash .github/scripts/test-duckdb-build-modes.sh - name: Verify selection, cancellation and aggregate admission run: | bash .github/scripts/test-collector-control-scheduling.sh @@ -259,6 +263,7 @@ jobs: CARGO_TERM_COLOR: always CARGO_TARGET_DIR: ${{ github.workspace }}/rust_hft/target CARGO_INCREMENTAL: "0" + MONDAY_CARGO_TARGET_LAYOUT: owning-workspace-v1 CARGO_PROFILE_DEV_DEBUG: "0" CARGO_PROFILE_TEST_DEBUG: "0" steps: @@ -275,21 +280,27 @@ jobs: - name: Install Linux build tools run: bash "$GITHUB_WORKSPACE/.github/workflows/install-ubuntu-packages.sh" clang mold protobuf-compiler ruby + - name: Prepare committed native source before cache restore + run: bash "$GITHUB_WORKSPACE/.github/scripts/prepare-research-native-source.sh" + - name: Record cache dimensions id: cache-info - shell: bash - run: echo "rust=$(rustc -Vv | awk '/^release:/ {print $2}')" >> "$GITHUB_OUTPUT" + env: + MONDAY_CI_CACHE_PLAN: ${{ toJSON(needs.scope.outputs) }} + run: | + native=$(bash "$GITHUB_WORKSPACE/.github/scripts/ci-owner-cache.sh" native-input) + coverage=$(bash "$GITHUB_WORKSPACE/.github/scripts/ci-owner-cache.sh" coverage-input) + export MONDAY_CI_CACHE_NATIVE=${native#native=} + export MONDAY_CI_CACHE_COVERAGE=${coverage#coverage=} + bash "$GITHUB_WORKSPACE/.github/scripts/ci-owner-cache.sh" cache-input >> "$GITHUB_OUTPUT" - name: Cache Rust - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + id: owner-cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: - # The archive builds research-core. Cleanup must use that owner's - # dependency graph, including native DuckDB compilation outputs. - workspaces: | - rust_hft/research-core -> ../target - key: loop-nextest-research-owner-v2-${{ steps.cache-info.outputs.rust }} - cache-all-crates: false - save-if: ${{ github.ref == 'refs/heads/main' }} + path: ${{ steps.cache-info.outputs.cache_paths }} + key: loop-nextest-${{ steps.cache-info.outputs.key }} + restore-keys: loop-nextest-${{ steps.cache-info.outputs.compat_prefix }} - name: Install cargo-nextest run: bash "$GITHUB_WORKSPACE/.github/scripts/install-cargo-nextest.sh" @@ -322,6 +333,17 @@ jobs: if-no-files-found: error retention-days: 7 + - name: Retain dependencies for the trusted archive cache + if: success() + run: bash "$GITHUB_WORKSPACE/.github/scripts/ci-owner-cache.sh" cleanup + + - name: Save trusted archive dependencies + if: ${{ github.ref == 'refs/heads/main' && success() && steps.owner-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ${{ steps.cache-info.outputs.cache_paths }} + key: loop-nextest-${{ steps.cache-info.outputs.key }} + rust_loop_nextest_shard: name: Loop nextest shard (${{ matrix.shard }}) needs: rust_loop_nextest_archive @@ -424,6 +446,10 @@ jobs: if: ${{ (needs.scope.outputs.toolchain == 'true') }} run: bash "$GITHUB_WORKSPACE/.github/workflows/install-ubuntu-packages.sh" clang mold protobuf-compiler + - name: Prepare committed native source before cache restore + if: ${{ needs.scope.outputs.toolchain == 'true' }} + run: bash "$GITHUB_WORKSPACE/.github/scripts/prepare-research-native-source.sh" + - name: Record cache dimensions if: ${{ (needs.scope.outputs.toolchain == 'true') }} id: cache-info diff --git a/docs/research/CEX_IMAGE_BUILD_SCOPE.md b/docs/research/CEX_IMAGE_BUILD_SCOPE.md new file mode 100644 index 000000000..82ec6c82c --- /dev/null +++ b/docs/research/CEX_IMAGE_BUILD_SCOPE.md @@ -0,0 +1,88 @@ +# CEX image build scope + +The production CEX image packages eight executables through five release recipes. +The offline diagnostic has a separate local build documented in [OFFLINE_BASELINE.md](OFFLINE_BASELINE.md). +Removing that executable removes its operator-plus-offline recipe from every CEX image build. +It does not remove the scientific worker's offline-diagnostic library feature. + +## Entry points and dependency boundaries + +| Entry | Build owner and features | Packaging decision | +| --- | --- | --- | +| `hft-backtest` | research-core; default | Retain the existing Docker entry point. | +| `alpha-harness` | research-core; default operator | Retain Campaign freeze, finalize and dispatch commands. Controller shares these exact bytes. | +| `monday-cex-worker` | research-core; scientific, without alpha-harness default features | Retain the admitted Campaign execution entry point. | +| `monday-offline-baseline` | Explicit local alpha-harness offline-diagnostic build, without defaults | Exclude from production images. No production entry point invokes it. | +| `lob-pit-materializer` | data-pipelines; collector | Retain materialization script dependency. | +| `binance-market-tape-slicer` | data-pipelines; collector | Retain materialization script dependency. | +| `binance-replay-parquet-materializer` | data-pipelines; collector | Retain materialization script dependency. | +| `clickhouse-analytics-materializer` | data-pipelines; collector | Retain data preparation tooling. | +| `monday-cex-prepare` | data-pipelines; preparation | Retain preparation and its source-version smoke check. | + +The catalog and both Docker build paths must agree. Product tests check the exact +prebuilt copies, producer archive, binary startup and compiler failure handling. +The existing dependency graph still propagates shared source changes conservatively. +No source path is excluded from CI by this change. + +## Reuse and identity + +Parameter-only experiments reuse an admitted immutable Build and verified prepared inputs. +Data semantics, including label windows, still invalidate prepared-input reuse. +See [CAMPAIGN_PREPARATION_REUSE.md](CAMPAIGN_PREPARATION_REUSE.md). + +The release profile, target, compiler, native environment and workspace inputs remain +bound to compilation provenance. Cargo targets stay isolated by owning workspace. +The changed recipe list remains part of exact build inputs and artifact validation. +The dependency restore identity binds the small compiler helper, cache cleanup, +compiler, native environment, flags and owning-workspace profiles. +Cargo metadata resolves DuckDB's version, immutable source, features and dependency +closure. Native closure changes isolate the dependency cache. +Business source, unrelated resolved packages, packaging and smoke scripts remain +outside this identity. Complete recipe provenance still binds those scripts. +This migration starts a new key. Branch CI cannot populate trusted main caches. +The historical main run restored dependencies and marked DuckDB artifacts fresh; +its 13m48s recipe phase does not establish that every main build was cold. +Dependency cache reuse does not authenticate executable bytes: builds still produce, +hash and smoke-test the complete selected product at the admitted source revision. +No research-profile executable is presented as a release artifact. + +## Measurement and remaining scope + +The prior reference is Monorepo 38429350564 at 14m19s and Prediction 38429350500 +at 20m51s. Research job 115408859232 took about 15m57s, including 13m52s build. +The reported merge-to-ACR interval was 24m43s. These are historical observations, +not a controlled benchmark for this change. + +This change reduces nine executables and six recipes to eight and five. +It does not promise a proportional wall-time reduction: owners run concurrently, +and the scientific worker still compiles its fitting and offline library closure. +Use retained recipe timings, exact cache-hit state and runner identity to compare runs. +Report queue, compilation, packaging and publication separately; overlapping phases +must not be added together. A draft PR provides no merge-to-ACR measurement. + +Further separation of preparation, operator and scientific images requires a complete +consumer and signed-product protocol change. This patch preserves that boundary. +The 10–15 minute small-change target remains unproven until comparable CI measurements exist. + +The existing diagnostic run 38525297670 tested commit +`4e64020efbda664eef46790de8373c4b6c5368aa` on one EPYC 9V45 runner with four CPUs. +Its cold recipe phase took 1481s. Full local reuse took 2s. +After removing local compilation bytes, serial rebuilding took 568s and parallel +rebuilding took 502s. DuckDB artifacts stayed fresh in both dependency-warm phases; +local packages rebuilt and the binary hashes agreed. +These are recipe timings, not full CI or publication timings. +The runner differs from the historical EPYC 7763 baseline. This is not an old/new +code speedup estimate. The dependency identity patch still needs its own CI validation. + +## Native resolution deduplication + +The catalog repeats Cargo metadata requests for recipes with identical workspace and feature arguments. +Resolve each distinct command once per identity capture. Keep package-qualified feature requests separate. +This reduces eight catalog requests to four, before adding the independent publisher request. +Regression tests compare the resulting native identity with separate per-recipe resolution. +The memo exists only inside one process. It cannot reuse metadata after source or lockfile changes. + +This reduces metadata overhead, not Rust compilation. The helper remains part of the compatibility key. +Its first CI execution therefore needs a new cache identity. Only successful main builds save trusted bytes. +A PR rerun cannot establish cross-run warm-cache performance before that identity exists on main. +Do not interpret same-run diagnostic phases as cross-run cache evidence. diff --git a/docs/research/DUCKDB_CACHE_AUDIT_20261016.md b/docs/research/DUCKDB_CACHE_AUDIT_20261016.md new file mode 100644 index 000000000..daa1e9204 --- /dev/null +++ b/docs/research/DUCKDB_CACHE_AUDIT_20261016.md @@ -0,0 +1,185 @@ +# DuckDB cache and build budget audit + +Evidence date: 2026-10-16. Times below are durations. GitHub timestamps use UTC. +PR: [#1430](https://github.com/proerror77/monday/pull/1430). +Reviewed branch head: `22c909b727c3f7f5e365b304925844e3f14dcf81`. +This includes the fingerprint diagnostics added after `073396357`. + +## Actual run evidence + +| Run | Source tested | Cache result | Recipe phase | Complete Prediction CI | +| --- | --- | --- | --- | --- | +| [38429350500](https://github.com/proerror77/monday/actions/runs/38429350500) | main `08b98142` | Previous-run cache restored; DuckDB build script and library fresh | 13m48s | 20m51s | +| [38767757268](https://github.com/proerror77/monday/actions/runs/38767757268) | main `7de64784` | Previous-run cache restored; first DuckDB recipe not fresh | 26m17s | 30m14s | +| [38831045248](https://github.com/proerror77/monday/actions/runs/38831045248) | PR merge `0be6aab8`, head `07339635` | Exact and compatible keys absent | 36m30s | 39m35s | +| [38836120877](https://github.com/proerror77/monday/actions/runs/38836120877) | PR merge `70eccba8`, head `22c909b7` | Exact and compatible keys absent | 26m01s | 29m19s | + +The last run used EPYC 9V45 with four CPUs. Its image job took 27m24s. +The build step took 26m04s. The operator recipe took 19m21s. +The scientific worker took 4m49s and marked both DuckDB artifacts fresh. +The offline smoke step took 48s. Steps overlap or nest; do not add their durations. +The preceding PR run used EPYC 7763. These are not controlled speed comparisons. + +## What directly caused the latest native compilation + +Run 38836120877 logged failed reads for all three DuckDB fingerprints: + +```text +target/research-core/x86_64-unknown-linux-gnu/release/.fingerprint/libduckdb-sys-52f9d69926e21174/lib-libduckdb_sys +target/research-core/x86_64-unknown-linux-gnu/release/.fingerprint/libduckdb-sys-bdeb33e435cea0fa/run-build-script-build-script-build +target/research-core/release/.fingerprint/libduckdb-sys-e0e6c3cde5c60cd6/build-script-build-script-build +``` + +The restore log also says `Cache not found`. This directly establishes a cold +target, rather than a dirty fingerprint caused by a dependency update. +Both PR runs requested compatibility identity +`11f7e6516ebdf509e52dba420622288cdae7e4546fb6ffa5ab64c3b01e7adf49`. +The instrumentation-only change preserved that identity. + +Historical main runs used compatibility identity +`875c4a7d72813afb1feadc78bb76e0876d08ac727ec9f1ad648ef51ff634c37c`. +The identity migration intentionally separates those caches. +Successful main builds alone save cleaned dependency bytes. PR runs do not save them. +Therefore repeated PR runs cannot populate the new trusted identity. +Do not broaden writes to untrusted PR contexts to obtain a warm measurement. + +## Build dependencies and real Cargo units + +The downloaded `libduckdb-sys 1.10504.0` registry archive matched the lock checksum: +`94e67523a40ee3da30411e00e243990b3760d91877202ab2f39c03ab34f8dbfc`. +Its manifest unconditionally declares reqwest with blocking and rustls-tls features. +Only the non-bundled source module uses the download client. + +Locked metadata at tested source `70eccba8` contains the dependency chain: + +```text +libduckdb-sys 1.10504.0 build script -> reqwest 0.12.28 -> rustls 0.23.46 +``` + +A read-only Cargo 1.98.1 unit graph confirms a host reqwest/rustls build beneath +the bundled DuckDB build script. This is an actual build dependency, not only +an unused lockfile entry. The graph command used `RUSTC_BOOTSTRAP=1` only for +the diagnostic `-Z unstable-options --unit-graph` operation. It compiled no +production bytes and changed no workflow, lockfile, cache admission or compiler flags. +Production builds continue to use the existing stable Cargo command. + +Security commit `7fbeb700` changes rustls from 0.23.45 to 0.23.46. +That identity change can legitimately invalidate the DuckDB build script and its native outputs. +Run 38767757268 restored the exact cache saved by run 38429350500, then rebuilt DuckDB. +However, it did not retain Cargo dirty reasons. The dependency change supports +this explanation; the historical invalidation reason cannot be recovered from those logs. +Do not claim it was the sole reason. + +A separate three-package Cargo 1.98.1 experiment tested this mechanism. +A bundled C archive had a download build dependency referenced only under the disabled cfg branch. +Cold build: native build script/library fresh=false. Unchanged build: both fresh=true. +Changing only the unused build dependency made both fresh=false and reran the C build. +Cargo reported `StaleDepFingerprint`; the archive hash stayed identical across all three phases. +The consumer executed successfully each time. This is a small mechanism reproduction, +not a DuckDB compilation benchmark or historical-run fingerprint evidence. + +## Conservative boundaries + +The exact suffix binds full provenance. Source and packaging changes can alter it +without preventing compatible dependency restoration. +The compatible prefix binds the compiler, builder digest, native environment, +flags, profiles, compiler helper, cleanup contract and native dependency identities. +Final executable hashes, source admission, ABI checks and offline smoke remain separate. + +There is still excess scope: + +- Native environment capture hashes every installed dpkg package, including packaging tools. +- One combined prefix covers the cached owners. A native change can prevent restoration for all owners. +- Cargo metadata resolves workspace feature unions rather than exact host compilation units. + At `70eccba8`, metadata adds reqwest json, stream, h2 and http2 features. + The DuckDB host reqwest unit uses blocking/rustls without those extra features. + The metadata closure contains 137 package identities; the host build closure contains 113. + The excess includes quinn, h2 and their dependencies. This proves excess key scope, + not an observed cache miss caused by any one of those packages. +- The native identity helper and cache-layout script bytes enter the compiler contract hash. + A semantics-preserving helper edit still migrates the identity. + +These are potential miss amplifiers. They are not dirty reasons for the latest cold run. +Removing rustls from the identity alone would not stop Cargo from rebuilding it. +It would also discard a real build-dependency identity boundary. +This change therefore retains the native key, checksum and feature controls. + +Safe future narrowing needs either an authenticated native artifact boundary or +a reviewed upstream change that makes download dependencies conditional. +The latter must preserve non-bundled behavior and regenerate every owning lockfile. +Exact host-unit projection also needs a supported producer interface and regression evidence. +The diagnostic unstable Cargo interface is not added to production key generation. + +## CPU allocation correction + +The old scheduler assigned two jobs to each owner on a four-CPU runner. +That allocation remained fixed after both faster owners exited. +In run 38831045248, Prediction finished at 00:59:59 UTC. +The research-core owner then ran alone until 01:21:07 UTC with two jobs. +This is 21m08s of nominal two-job capacity left unassigned. +It does not prove the CPUs were physically idle or quantify recoverable time. + +The correction grants the final owner the admitted total budget only after every +sibling exits and the pending owner queue is empty. Each recipe reads its budget +once before Cargo starts. An atomic file replacement prevents partial reads. +Current Cargo/cc jobservers retain their initial allocation. +Therefore this correction does not accelerate an already-running DuckDB build. +It can reclaim CPUs for the following scientific-worker recipe. + +The global CPU budget, memory overlap limit and one writer per owning target remain enforced. +Regression tests exercise a delayed owner, reclaim on its next recipe, total active +jobs, failure cancellation, TERM-ignoring descendants and removal of budget files. +The compiler helper and native compatibility identity remain unchanged. +These tests establish scheduling behavior, not elapsed-time improvement. + +## Remaining 10–15 minute conditions + +1. Obtain a compatible, successful trusted producer cache and verify restoration + in a different run. Do not label same-run reuse as cross-run evidence. +2. Capture dirty reasons on a genuine restored-but-rebuilt case. Distinguish + missing files, build-dependency units, features, compiler/environment and file freshness. +3. Measure the CPU correction on comparable hardware and cache state. It cannot + reclaim CPUs inside an already-running native build. +4. Budget the full critical path: admission, setup, build, archive transfer, smoke + and final CI gate. Historical warm Prediction CI still exceeded 15 minutes. +5. Measure publication separately only under an authorized publication task. + A successful PR CI run does not establish merge-to-registry or production latency. + +Cold native compilation already exceeds the complete target on the measured runners. +The existing 8m22s same-run dependency-warm recipe result is not complete CI. +No measured result here establishes a 10–15 minute full delivery outcome. + +## Download isolation implementation + +The continuation starts from PR head `f82a1ad2` and merges main `4310dc7e`. +The patch vendors the checksum-verified `libduckdb-sys 1.10504.0` package at +`rust_hft/native/libduckdb-sys`. Its C++ archive and native compiler backends match upstream bytes. Bindings keep +upstream code with one trailing blank line removed. The functional changes affect +only its manifest and downloader cfg gates. +The license and patch provenance remain beside the source. + +Research-core and data-pipelines select this patch. Their lockfiles retain all +other package versions. Bundled and system linking exclude reqwest and zip. +The `download` feature retains the upstream downloader and requires the existing +`DUCKDB_DOWNLOAD_LIB` switch. Requesting a download without that feature fails. +Prediction retains its independently locked upstream `1.10501.0` package. + +Research cache identities hash every native patch file and the resolved native +closure. The dev/test owner cache also binds patch bytes and source preparation. +Both cleaners retain this exact native path while deleting application artifacts. +Loop nextest uses the same owner-cache contract, with its own namespace. This +avoids Swatinem removing the newly local native dependency before a trusted save. +PR jobs still cannot save trusted dependency caches. + +Fresh CI checkouts normalize native source mtimes before any target restore. +Preparation rejects dirty source or existing owning targets. Local builds retain +actual mtimes. This handles local-path freshness without hiding changed source +bytes from the compatibility identity. Native library ABI, flags, compiler, +profiles, artifact hashing and offline smoke remain enforced. + +Focused checks cover system/downloader compilation, rejection without the +feature, bundled dependency exclusion, source-byte cache invalidation, cleanup +retention and source timestamp guards. Actual macOS integration checks use +`SELECT 42` with the explicit downloader and system linking. The bundled native +build and complete existing CI are reported separately; fixture and dependency +graph checks do not establish a 10–15 minute full pipeline. diff --git a/docs/research/OFFLINE_BASELINE.md b/docs/research/OFFLINE_BASELINE.md index 2d095f0bb..4b7b7a5e1 100644 --- a/docs/research/OFFLINE_BASELINE.md +++ b/docs/research/OFFLINE_BASELINE.md @@ -2,6 +2,9 @@ The Rust `monday-offline-baseline` driver fits fixed Ridge folds on prepared native rows and evaluates their full rolling out-of-sample ledger. This diagnostic cannot complete a production Campaign or grant compute authority. +The production `cex-runner` image excludes this diagnostic executable. Build it explicitly with the recipe below. +Its local output is not a signed production Build or an admitted Campaign runner. + ## Reproduce a local synthetic run From the repository root: diff --git a/rust_hft/data-pipelines/Cargo.lock b/rust_hft/data-pipelines/Cargo.lock index 01b473709..568a96fb2 100644 --- a/rust_hft/data-pipelines/Cargo.lock +++ b/rust_hft/data-pipelines/Cargo.lock @@ -4014,18 +4014,14 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libduckdb-sys" version = "1.10504.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94e67523a40ee3da30411e00e243990b3760d91877202ab2f39c03ab34f8dbfc" dependencies = [ "cc", "flate2", "pkg-config", - "reqwest 0.12.28", "serde", "serde_json", "tar", "vcpkg", - "zip", ] [[package]] diff --git a/rust_hft/data-pipelines/Cargo.toml b/rust_hft/data-pipelines/Cargo.toml index 62c0a01e3..3ef23c4f3 100644 --- a/rust_hft/data-pipelines/Cargo.toml +++ b/rust_hft/data-pipelines/Cargo.toml @@ -57,4 +57,5 @@ codegen-units = 1 debug = false [patch.crates-io] +libduckdb-sys = { path = "../native/libduckdb-sys" } rust_decimal = { git = "https://github.com/proerror77/monday-rust-decimal-security-694", rev = "4f7bc6f02d7920b6416988914ae86d25f6670b22" } diff --git a/rust_hft/deployment/docker/Dockerfile.research b/rust_hft/deployment/docker/Dockerfile.research index 998f11fd6..6122ebb1e 100644 --- a/rust_hft/deployment/docker/Dockerfile.research +++ b/rust_hft/deployment/docker/Dockerfile.research @@ -44,10 +44,6 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ && MONDAY_SOURCE_REVISION="$SOURCE_REVISION" cargo --config 'build.rustc-wrapper=""' build \ --manifest-path data-pipelines/Cargo.toml --release --locked \ -p hft-cex-data-preparation --bin monday-cex-prepare \ - && MONDAY_SOURCE_REVISION="$SOURCE_REVISION" cargo --config 'build.rustc-wrapper=""' build \ - --manifest-path research-core/Cargo.toml --release --locked \ - -p alpha-harness --features offline-diagnostic --bin monday-offline-baseline \ - && install -D -m 0755 /work/target/release/monday-offline-baseline /out/bin/monday-offline-baseline \ && install -D -m 0755 /work/target/release/monday-cex-prepare /out/bin/monday-cex-prepare \ && install -D -m 0755 /work/target/release/hft-backtest /out/bin/hft-backtest \ && install -D -m 0755 /work/target/release/alpha-harness /out/bin/alpha-harness \ @@ -78,8 +74,6 @@ ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/hft-backtest"] FROM runtime-base AS prebuilt -COPY --chmod=0755 research-bin/monday-offline-baseline /usr/local/bin/monday-offline-baseline - COPY --chmod=0755 research-bin/monday-cex-prepare /usr/local/bin/monday-cex-prepare COPY --chmod=0755 research-bin/hft-backtest /usr/local/bin/hft-backtest diff --git a/rust_hft/native/libduckdb-sys/Cargo.toml b/rust_hft/native/libduckdb-sys/Cargo.toml new file mode 100644 index 000000000..44abbe03f --- /dev/null +++ b/rust_hft/native/libduckdb-sys/Cargo.toml @@ -0,0 +1,155 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +rust-version = "1.85.1" +name = "libduckdb-sys" +version = "1.10504.0" +authors = [ + "DuckDB Labs and contributors", + "wangfenjin ", +] +build = "build.rs" +links = "duckdb" +exclude = ["duckdb-sources"] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Native bindings to the libduckdb library, C API" +homepage = "https://github.com/duckdb/duckdb-rs" +readme = "README.md" +keywords = [ + "duckdb", + "database", + "ffi", +] +categories = [ + "external-ffi-bindings", + "database", +] +license = "MIT" +repository = "https://github.com/duckdb/duckdb-rs" + +[workspace] + +[features] +download = ["dep:reqwest", "dep:zip"] +autocomplete = ["bundled-cmake"] +buildtime_bindgen = [ + "bindgen", + "pkg-config", + "vcpkg", +] +bundled = ["cc"] +bundled-cmake = [ + "bundled", + "dep:cmake", + "dep:which", + "parquet", +] +default = [ + "vcpkg", + "pkg-config", +] +extensions-full = [ + "json", + "parquet", +] +icu = ["bundled-cmake"] +json = ["bundled"] +loadable-extension = [ + "prettyplease", + "quote", + "syn", +] +parquet = ["bundled"] +tpcds = ["bundled-cmake"] +tpch = ["bundled-cmake"] +winduckdb = [] + +[lib] +name = "libduckdb_sys" +path = "src/lib.rs" + +[dev-dependencies.arrow] +version = "58" +features = ["ffi"] +default-features = false + +[build-dependencies.bindgen] +version = "0.72.1" +features = ["runtime"] +optional = true +default-features = false + +[build-dependencies.cc] +version = "1.0" +features = ["parallel"] +optional = true + +[build-dependencies.cmake] +version = "0.1" +optional = true + +[build-dependencies.flate2] +version = "1.0" + +[build-dependencies.pkg-config] +version = "0.3.24" +optional = true + +[build-dependencies.prettyplease] +version = "0.2.20" +optional = true + +[build-dependencies.quote] +version = "1.0.21" +optional = true + +[build-dependencies.reqwest] +optional = true +version = "0.12" +features = [ + "blocking", + "rustls-tls", +] +default-features = false + +[build-dependencies.serde] +version = "1.0" +features = ["derive"] + +[build-dependencies.serde_json] +version = "1.0" + +[build-dependencies.syn] +version = "2.0.15" +optional = true + +[build-dependencies.tar] +version = "0.4.38" + +[build-dependencies.vcpkg] +version = "0.2" +optional = true + +[build-dependencies.which] +version = "8" +optional = true + +[build-dependencies.zip] +optional = true +version = "6" +features = ["deflate"] +default-features = false diff --git a/rust_hft/native/libduckdb-sys/LICENSE b/rust_hft/native/libduckdb-sys/LICENSE new file mode 100644 index 000000000..ccccdf5f0 --- /dev/null +++ b/rust_hft/native/libduckdb-sys/LICENSE @@ -0,0 +1,7 @@ +Copyright 2021-2026 Stichting DuckDB Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/rust_hft/native/libduckdb-sys/MONDAY-PATCH.md b/rust_hft/native/libduckdb-sys/MONDAY-PATCH.md new file mode 100644 index 000000000..118d47b76 --- /dev/null +++ b/rust_hft/native/libduckdb-sys/MONDAY-PATCH.md @@ -0,0 +1,22 @@ +# Monday download dependency isolation + +Upstream: `libduckdb-sys 1.10504.0`, MIT, from crates.io. +Registry archive SHA-256: +`94e67523a40ee3da30411e00e243990b3760d91877202ab2f39c03ab34f8dbfc`. + +The native source archive and compiler backends are unchanged. +Bindings retain upstream code; one generated file loses a trailing blank line. +`download` explicitly enables reqwest and zip. System linking and bundled +compilation do not compile these dependencies. Set both the feature and +`DUCKDB_DOWNLOAD_LIB=1` to use the original upstream downloader. Without the +feature, that environment variable fails with an actionable error. +Bundled mode retains precedence over the download environment variable. + +The owning research-core and data-pipelines manifests select this patch. +Prediction keeps its distinct locked upstream version. +The native cache identity binds every patch file, features, dependency checksums, +compiler, ABI, flags and profiles. Only this exact native path is retained during +cache cleanup. Application path dependencies are still removed. +Fresh CI checkouts normalize committed native source timestamps before cache +lookup. Complete source hashes isolate changed bytes. Preparation rejects dirty +source and existing owning targets. Local builds retain actual timestamps. diff --git a/rust_hft/native/libduckdb-sys/README.md b/rust_hft/native/libduckdb-sys/README.md new file mode 100644 index 000000000..020d7d093 --- /dev/null +++ b/rust_hft/native/libduckdb-sys/README.md @@ -0,0 +1,342 @@ +# duckdb-rs + +[![Latest Version](https://img.shields.io/crates/v/duckdb.svg)](https://crates.io/crates/duckdb) +[![Documentation](https://img.shields.io/badge/docs.rs-duckdb-orange)](https://docs.rs/duckdb) +[![MIT License](https://img.shields.io/crates/l/duckdb.svg)](LICENSE) +[![Downloads](https://img.shields.io/crates/d/duckdb.svg)](https://crates.io/crates/duckdb) +[![CI](https://github.com/duckdb/duckdb-rs/workflows/CI/badge.svg)](https://github.com/duckdb/duckdb-rs/actions) + +duckdb-rs is an ergonomic Rust wrapper for [DuckDB](https://github.com/duckdb/duckdb). + +You can use it to: + +- Query DuckDB with type-safe bindings and an API inspired by [rusqlite](https://github.com/rusqlite/rusqlite). +- Read and write Arrow, Parquet, JSON, and CSV formats natively. +- Create DuckDB extensions in Rust with custom scalar and table functions. + +## Quickstart + +Create a new project and add the `duckdb` crate: + +```shell +cargo new quack-in-rust +cd quack-in-rust +cargo add duckdb -F bundled +``` + +Update `src/main.rs` with the following code: + +```rust +use duckdb::{params, Connection, Result}; + +struct Duck { + id: i32, + name: String, +} + +fn main() -> Result<()> { + let conn = Connection::open_in_memory()?; + + conn.execute( + "CREATE TABLE ducks (id INTEGER PRIMARY KEY, name TEXT)", + [], // empty list of parameters + )?; + + conn.execute_batch( + r#" + INSERT INTO ducks (id, name) VALUES (1, 'Donald Duck'); + INSERT INTO ducks (id, name) VALUES (2, 'Scrooge McDuck'); + "#, + )?; + + conn.execute( + "INSERT INTO ducks (id, name) VALUES (?, ?)", + params![3, "Darkwing Duck"], + )?; + + let ducks = conn + .prepare("FROM ducks")? + .query_map([], |row| { + Ok(Duck { + id: row.get(0)?, + name: row.get(1)?, + }) + })? + .collect::>>()?; + + for duck in ducks { + println!("{}) {}", duck.id, duck.name); + } + + Ok(()) +} +``` + +Execute the program with `cargo run` and watch DuckDB in action! + +## Examples + +The following [examples](crates/duckdb/examples) demonstrate various features and use cases of duckdb-rs: + +- **basic** - Basic usage including creating tables, inserting data, and querying with and without Arrow. +- **appender** - Bulk data insertion using the appender API with transactions. +- **parquet** - Reading Parquet files directly using DuckDB's Parquet extension. +- **vscalar** - Defining and registering a custom scalar function (requires the `vscalar` feature). +- **vtab** - Defining a custom table function returning primitive, LIST, and STRUCT columns (requires the `vtab` feature). +- **repl** - Interactive SQL REPL. +- **hello-ext** - A DuckDB extension written in Rust (see [Building and loading extensions](#building-and-loading-extensions)). + +Run any example with `cargo run --example `. + +## Feature flags + +The `duckdb` crate provides a number of Cargo features that can be enabled to add functionality: + +### Virtual tables and functions + +- `vtab` - Base support for creating custom table functions and virtual tables. +- `vtab-arrow` - Apache Arrow integration for virtual tables. Enables conversion between Arrow RecordBatch and DuckDB data chunks. +- `vtab-excel` - Deprecated no-op retained for feature compatibility. +- `vscalar` - Create custom scalar functions that operate on individual values or rows. +- `vscalar-arrow` - Arrow-optimized scalar functions for vectorized operations. + +### File formats + +- `json` - Enables reading and writing JSON files. Implies `bundled`. +- `parquet` - Enables reading and writing Parquet files. Implies `bundled`. + +### Bundled DuckDB extensions + +These extensions are only available through the CMake build backend and imply `bundled-cmake`. + +- `autocomplete` - DuckDB's autocomplete extension. +- `icu` - DuckDB's ICU extension for locale-aware operations. +- `tpch` - DuckDB's TPC-H benchmark extension. +- `tpcds` - DuckDB's TPC-DS benchmark extension. + +### Data integration + +- `appender-arrow` - Efficient bulk insertion of Arrow data into DuckDB tables. +- `polars` - Integration with Polars DataFrames. + +### Convenience features + +- `vtab-full` - Enables virtual table features: `vtab-arrow` and `appender-arrow`; retains the deprecated `vtab-excel` compatibility flag. +- `extensions-full` - Enables all major extensions: `json`, `parquet`, and `vtab-full`. +- `modern-full` - Enables modern Rust ecosystem integrations: `chrono`, `serde_json`, `url`, `r2d2`, `uuid`, and `polars`. + +### Build configuration + +- `bundled` - Uses a bundled version of DuckDB's source code and compiles it during build. This is the simplest way to get started and avoids needing DuckDB system libraries. +- `bundled-cmake` - _Experimental_. Builds DuckDB via its upstream CMake build system instead of `cc`. Requires a duckdb-rs checkout (not available from crates.io). See [step 2](#notes-on-building-duckdb-and-libduckdb-sys) below for details. +- `buildtime_bindgen` - Use bindgen at build time to generate fresh bindings instead of using pre-generated ones. +- `loadable-extension` - _Experimental_ support for creating loadable DuckDB extensions. Includes procedural macros for extension development. Only enable this when building an extension, never for a client application - see [Database client or extension?](#database-client-or-extension) + +## Installation + +### Versioning + +Starting with DuckDB `v1.5.0`, the duckdb-rs version encodes the DuckDB version in its second semver component. +The format is `1.MAJOR_MINOR_PATCH.x`, e.g., DuckDB `v1.5.0` maps to duckdb-rs `1.10500.x`. +Use a tilde requirement to receive duckdb-rs patch releases without automatically moving to a different bundled DuckDB version. + +### Using stable releases from crates.io + +The recommended way to use duckdb-rs is to add it from crates.io: + +```shell +cargo add duckdb -F bundled +``` + +Or manually add it to your `Cargo.toml`: + +```toml +[dependencies] +duckdb = { version = "~1.10504.0", features = ["bundled"] } +``` + +### Using the development version from git + +To use the latest development version from the main branch, you can specify a git dependency in your `Cargo.toml`: + +```toml +# Use a specific branch +duckdb = { git = "https://github.com/duckdb/duckdb-rs", branch = "main", features = ["bundled"] } + +# Use a specific commit +duckdb = { git = "https://github.com/duckdb/duckdb-rs", rev = "abc123def", features = ["bundled"] } +``` + +Note: Using the main branch of duckdb-rs means you'll get the latest Rust bindings and features, but you'll still be using whatever version of DuckDB core is bundled with that commit (when using the `bundled` feature). + +## Notes on building duckdb and libduckdb-sys + +`libduckdb-sys` is a separate crate from `duckdb-rs` that provides the Rust +declarations for DuckDB's C API. By default, `libduckdb-sys` attempts to find a DuckDB library that already exists on your system using pkg-config, or a +[Vcpkg](https://github.com/Microsoft/vcpkg) installation for MSVC ABI builds. + +You can adjust this behavior in a number of ways: + +1. If you use the `bundled` feature, `libduckdb-sys` will use the + [cc](https://crates.io/crates/cc) crate to compile DuckDB from source and + link against that. This source is embedded in the `libduckdb-sys` crate and + tracks the DuckDB version vendored for that duckdb-rs release. + This is probably the simplest solution to any build problems. You can enable this by adding the following in your `Cargo.toml` file: + + ```bash + cargo add duckdb --features bundled + ``` + + `Cargo.toml` will be updated. + + ```toml + [dependencies] + duckdb = { version = "~1.10504.0", features = ["bundled"] } + ``` + +2. If you use the `bundled-cmake` feature, `libduckdb-sys` will build DuckDB from the local checkout in `crates/libduckdb-sys/duckdb-sources` using upstream CMake. This keeps plain `bundled` unchanged while allowing CMake-only extensions such as `icu`. + + Example: + + ```toml + [dependencies] + duckdb = { git = "https://github.com/duckdb/duckdb-rs", branch = "main", features = ["bundled-cmake", "icu"] } + ``` + + Notes: + - `bundled-cmake` is _experimental_ and requires a git/workspace checkout. Published crates omit the full `duckdb-sources` tree. + - It implies `bundled` for conditional-compilation gates, but uses CMake instead of the `cc` backend. Any CMake-only extension feature (e.g. `icu`) enables it automatically. + - It always links DuckDB's default static extensions (`core_functions` and `parquet`) and therefore implies the `parquet` Cargo feature. + - Plain `bundled` uses DuckDB's standard allocator and skips jemalloc. `bundled-cmake` enables upstream jemalloc on supported 64-bit, non-musl Linux targets. Other targets follow DuckDB's CMake checks. Set `DUCKDB_DISABLE_JEMALLOC=1` to force the standard allocator. + - Extension autoload/autoinstall are enabled to match `bundled`, despite upstream CMake defaults. Set `DUCKDB_DISABLE_EXTENSION_LOAD=1` or `DISABLE_EXTENSION_LOAD=1` to disable external extension install/load support and force autoload/autoinstall off. Statically linked extensions remain available. + - If `ninja` is on `PATH`, the build uses Ninja by default. Set `CMAKE_GENERATOR` to override. + - DuckDB builds in `Release` mode by default, even for Rust debug builds, avoiding DuckDB's much slower debug/sanitizer profile. Override with `DUCKDB_CMAKE_BUILD_TYPE` or `CMAKE_BUILD_TYPE`. `DUCKDB_CMAKE_BUILD_TYPE` takes precedence. + - `DUCKDB_EXTENSION_CONFIGS` is unsupported. Setting it fails fast. + - Use `cargo build -vv -F bundled-cmake` for CMake configure/build logs. + +3. When linking against a DuckDB library already on the system (so _not_ using any of the `bundled` features), you can set the `DUCKDB_LIB_DIR` environment variable to point to a directory containing the library. You can also set the `DUCKDB_INCLUDE_DIR` variable to point to the directory containing `duckdb.h`. + + Linux example: + + ```shell + wget https://github.com/duckdb/duckdb/releases/download/v1.5.4/libduckdb-linux-arm64.zip + unzip libduckdb-linux-arm64.zip -d libduckdb + + export DUCKDB_LIB_DIR=$PWD/libduckdb + export DUCKDB_INCLUDE_DIR=$DUCKDB_LIB_DIR + export LD_LIBRARY_PATH=$DUCKDB_LIB_DIR + + cargo build --examples + ``` + + macOS example: + + ```shell + wget https://github.com/duckdb/duckdb/releases/download/v1.5.4/libduckdb-osx-universal.zip + unzip libduckdb-osx-universal.zip -d libduckdb + + export DUCKDB_LIB_DIR=$PWD/libduckdb + export DUCKDB_INCLUDE_DIR=$DUCKDB_LIB_DIR + export DYLD_FALLBACK_LIBRARY_PATH=$DUCKDB_LIB_DIR + + cargo build --examples + ``` + +4. Setting `DUCKDB_DOWNLOAD_LIB=1` makes the build script download pre-built DuckDB binaries from GitHub Releases. This always links against the dynamic library in the archive (setting `DUCKDB_STATIC` has no effect), and it effectively automates the manual steps above. The archives are cached in `target/duckdb-download//` and that directory is automatically added to the linker search path. The downloaded version always matches the DuckDB version encoded in the `libduckdb-sys` crate version. + + ```shell + DUCKDB_DOWNLOAD_LIB=1 cargo test + ``` + +5. Installing the DuckDB development packages will usually be all that is required, but + the build helpers for [pkg-config](https://github.com/alexcrichton/pkg-config-rs) + and [vcpkg](https://github.com/mcgoo/vcpkg-rs) have some additional configuration + options. The default when using vcpkg is to dynamically link, + which must be enabled by setting `VCPKGRS_DYNAMIC=1` environment variable before build. + +When none of the options above are used, the build script falls back to this discovery path and will emit the appropriate `cargo:rustc-link-lib` directives if DuckDB is found on your system. + +Cross-compiling with `bundled` is best-effort (not covered by CI): it compiles DuckDB from C++ source and so needs a working C++ cross-compiler for the target, not just the Rust toolchain. If you can't get one working, prefer one of the non-`bundled` options above (e.g. `DUCKDB_LIB_DIR` or `DUCKDB_DOWNLOAD_LIB=1`) and link a pre-built library for the target instead. + +### ICU extension and the bundled features + +When using the `bundled` feature, the ICU extension is not included due to crates.io's 10MB package size limit. This means some date/time operations (like `now() - interval '1 day'` or `ts::date` casts) will fail. You can load ICU at runtime: + +```rust,ignore +conn.execute_batch("INSTALL icu; LOAD icu;")?; +``` + +Alternatively, link against a system libduckdb that was compiled with ICU (see build instructions above). + +If you are working from a duckdb-rs checkout, you can also use `bundled-cmake,icu` to compile ICU in through DuckDB's CMake build. + +### Binding generation + +We use [bindgen](https://crates.io/crates/bindgen) to generate the Rust +declarations from DuckDB's C header file. `bindgen` +[recommends](https://github.com/servo/rust-bindgen#library-usage-with-buildrs) +running this as part of the build process for libraries that use it. We tried +this briefly (`duckdb` 0.10.0, specifically), but it had some annoyances: + +- The build time for `libduckdb-sys` (and therefore `duckdb`) increased + dramatically. +- Running `bindgen` requires a relatively recent version of Clang, which many + systems do not have installed by default. +- Running `bindgen` also requires the DuckDB header file to be present. + +So we try to avoid running `bindgen` at build time by shipping +pregenerated bindings for DuckDB. + +If you use the `bundled` feature, you will get pregenerated bindings for the +bundled version of DuckDB. If you want to run `bindgen` at build time to +produce your own bindings, use the `buildtime_bindgen` Cargo feature. + +## Thread safety + +A `Connection` is `Send` but not `Sync`: you may move it between threads, but you cannot share a single connection across threads at the same time. DuckDB does support concurrent access through _multiple_ connections to the same database (see [DuckDB's concurrency docs](https://duckdb.org/docs/current/connect/concurrency). + +For multi-threaded applications, give each thread its own connection. The simplest way is a connection pool via the `r2d2` feature: + +```rust,ignore +use duckdb::{DuckdbConnectionManager, params}; + +let manager = DuckdbConnectionManager::file("file.db")?; +let pool = r2d2::Pool::new(manager)?; + +// Each worker checks out its own connection from the pool. +let conn = pool.get()?; +conn.execute("INSERT INTO foo (bar) VALUES (?)", params![1])?; +``` + +## Database client or extension? + +duckdb-rs covers two distinct use cases, and you normally pick exactly one: + +- **As a database client** (most common): embed DuckDB in your Rust application to run queries, as shown in the [Quickstart](#quickstart). +- **For writing DuckDB extensions**: build a loadable `.duckdb_extension` that DuckDB loads at runtime. Enable the `loadable-extension` feature. + +Do **not** enable `loadable-extension` for a regular client application. It replaces DuckDB's normal C API functions with extension wrappers that are only initialized when DuckDB loads your extension. Outside an extension context, calls such as `Connection::open_in_memory()` then panic with `DuckDB API not initialized or DuckDB feature omitted`. + +## Building and loading extensions + +The [`hello-ext`](crates/duckdb/examples/hello-ext) example shows the extension API in action: it registers a `hello(name)` table function via the `loadable-extension` feature and the `duckdb_entrypoint_c_api` macro. + +`cargo build` alone does **not** produce a loadable extension though. DuckDB only loads files ending in `.duckdb_extension` that carry a metadata footer matching the target DuckDB version. A bare shared library fails with _The file is not a DuckDB extension. The metadata at the end of the file is invalid_. Appending that footer requires DuckDB's extension build tooling. + +So rather than wiring this up by hand, start from the official [extension-template-rs](https://github.com/duckdb/extension-template-rs) template for a complete, ready-to-build Rust extension. It handles the metadata step, platform/version detection, tests, and CI. Its `make debug` builds the shared library and then appends the footer, producing a `.duckdb_extension` you load with `duckdb -unsigned`. + +## Rust version compatibility + +duckdb-rs is built and tested with stable Rust, and keeps a rolling MSRV (minimum supported Rust version) that may only be updated when the encoded DuckDB major/minor version changes, on a need basis (e.g. project dependencies bump their MSRV or a particular Rust feature is useful for us). The new MSRV will be at least 6 months old. DuckDB patch updates and crate patch releases (e.g. `1.10500.0` to `1.10500.1`) are guaranteed to keep the same MSRV. + +## Contributing + +We welcome contributions! Take a look at [CONTRIBUTING.md](CONTRIBUTING.md) for more information. + +Join our [Discord](https://discord.gg/tcvwpjfnZx) to chat with the community in the #rust channel. + +## License + +Copyright (c) Stichting DuckDB Foundation + +Licensed under the [MIT license](LICENSE). diff --git a/rust_hft/native/libduckdb-sys/build.rs b/rust_hft/native/libduckdb-sys/build.rs new file mode 100644 index 000000000..325fe8a13 --- /dev/null +++ b/rust_hft/native/libduckdb-sys/build.rs @@ -0,0 +1,659 @@ +use std::{ + env, + path::{Path, PathBuf}, +}; + +/// Tells whether we're building for Windows. This is more suitable than a plain +/// `cfg!(windows)`, since the latter does not properly handle cross-compilation +/// +/// Note that there is no way to know at compile-time which system we'll be +/// targeting, and this test must be made at run-time (of the build script) See +/// https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-build-scripts +fn win_target() -> bool { + std::env::var("CARGO_CFG_WINDOWS").is_ok() +} + +/// Tells whether a given compiler will be used `compiler_name` is compared to +/// the content of `CARGO_CFG_TARGET_ENV` (and is always lowercase) +/// +/// See [`win_target`] +fn is_compiler(compiler_name: &str) -> bool { + std::env::var("CARGO_CFG_TARGET_ENV").is_ok_and(|v| v == compiler_name) +} + +fn main() { + let out_dir = env::var("OUT_DIR").unwrap(); + let out_path = Path::new(&out_dir).join("bindgen.rs"); + #[cfg(feature = "bundled")] + build_bundled_backend::main(&out_dir, &out_path); + #[cfg(not(feature = "bundled"))] + build_linked::main(&out_dir, &out_path) +} + +#[cfg(all(feature = "bundled", not(feature = "bundled-cmake")))] +mod build_bundled_cc; +#[cfg(all(feature = "bundled", feature = "bundled-cmake"))] +mod build_bundled_cmake; + +#[cfg(all(feature = "bundled", not(feature = "bundled-cmake")))] +use crate::build_bundled_cc as build_bundled_backend; +#[cfg(all(feature = "bundled", feature = "bundled-cmake"))] +use crate::build_bundled_cmake as build_bundled_backend; + +#[cfg(feature = "bundled")] +pub(crate) fn write_bindings(header_dir: &Path, out_path: &Path) { + #[cfg(feature = "buildtime_bindgen")] + { + use crate::{HeaderLocation, bindings}; + let header = HeaderLocation::IncludeDir(header_dir.to_path_buf()); + bindings::write_to_out_dir(header, out_path); + } + + #[cfg(not(feature = "buildtime_bindgen"))] + { + let _ = header_dir; + copy_pregenerated_bindings(out_path); + } +} + +/// Link the Windows system libraries DuckDB needs but that neither `cc` nor a +/// static CMake link adds automatically. Mirrors DuckDB's `DUCKDB_SYSTEM_LIBS` +/// (duckdb-sources/src/CMakeLists.txt) — keep in sync on submodule bumps. +/// Callers apply their own `win_target()` gate. +#[cfg(feature = "bundled")] +pub(crate) fn link_windows_system_libs() { + println!("cargo:rustc-link-lib=dylib=ws2_32"); + println!("cargo:rustc-link-lib=dylib=rstrtmgr"); + if is_compiler("msvc") { + println!("cargo:rustc-link-lib=dylib=bcrypt"); + } else { + println!("cargo:rustc-link-lib=dylib=stdc++"); + } +} + +#[cfg(not(feature = "buildtime_bindgen"))] +fn copy_pregenerated_bindings(out_path: &Path) { + let bindings_path = if is_loadable_extension() { + "src/bindgen_bundled_version_loadable.rs" + } else { + "src/bindgen_bundled_version.rs" + }; + println!("cargo:rerun-if-changed={bindings_path}"); + std::fs::copy(bindings_path, out_path).expect("Could not copy bindings to output directory"); +} + +pub enum HeaderLocation { + IncludeDir(PathBuf), + HeaderPath(PathBuf), + Wrapper, +} + +#[allow(dead_code)] +fn is_loadable_extension() -> bool { + cfg!(feature = "loadable-extension") +} + +#[allow(dead_code)] +fn header_filename() -> &'static str { + if is_loadable_extension() { + "duckdb_extension.h" + } else { + "duckdb.h" + } +} + +#[allow(dead_code)] +impl HeaderLocation { + fn from_env(lib_dir: &Path) -> Self { + if let Ok(include_dir) = env::var("DUCKDB_INCLUDE_DIR") { + return HeaderLocation::IncludeDir(PathBuf::from(include_dir)); + } + let header_path = lib_dir.join(header_filename()); + if header_path.exists() { + HeaderLocation::IncludeDir(lib_dir.to_path_buf()) + } else { + HeaderLocation::HeaderPath(header_path) + } + } + + #[cfg(feature = "buildtime_bindgen")] + fn header_path(&self) -> PathBuf { + match self { + HeaderLocation::IncludeDir(path) => path.join(header_filename()), + HeaderLocation::HeaderPath(path) => path.clone(), + HeaderLocation::Wrapper => PathBuf::from(if is_loadable_extension() { + "wrapper_ext.h" + } else { + "wrapper.h" + }), + } + } + + fn include_dir(&self) -> Option<&Path> { + match self { + HeaderLocation::IncludeDir(path) => Some(path), + HeaderLocation::HeaderPath(_) | HeaderLocation::Wrapper => None, + } + } +} + +#[cfg(not(feature = "bundled"))] +mod build_linked { + #[cfg(feature = "vcpkg")] + extern crate vcpkg; + + #[cfg(feature = "buildtime_bindgen")] + use super::bindings; + + use super::{HeaderLocation, is_compiler, is_loadable_extension, win_target}; + use std::{env, path::Path}; + #[cfg(feature = "download")] + use std::{fs, io, path::PathBuf}; + + pub fn main(out_dir: &str, out_path: &Path) { + // We need this to config the LD_LIBRARY_PATH + let header = find_duckdb(out_dir); + + // Publish the resolved include directory so downstream crates + // that compile their own C/C++ code can read it from + // `DEP_DUCKDB_INCLUDE`. Wrapper-only mode (no explicit dir + // from env / vcpkg / pkg-config / download) intentionally + // skips the emission — there's no single directory to point + // at; the system header is on the default search path already. + if let Some(include_dir) = header.include_dir() { + println!("cargo:include={}", include_dir.display()); + } + + #[cfg(not(feature = "buildtime_bindgen"))] + super::copy_pregenerated_bindings(out_path); + + #[cfg(feature = "buildtime_bindgen")] + bindings::write_to_out_dir(header, out_path); + } + + fn link_directive() -> &'static str { + // If the user specifies DUCKDB_STATIC, do static + // linking, unless it's explicitly set to 0. + match env::var("DUCKDB_STATIC") { + Ok(v) if v != "0" => "static=duckdb_static", + _ => "dylib=duckdb", + } + } + + fn emit_link_lib(link_directive: &str) { + if !is_loadable_extension() { + println!("cargo:rustc-link-lib={link_directive}"); + } + } + + #[cfg(feature = "pkg-config")] + fn duckdb_pkg_config() -> pkg_config::Config { + let mut config = pkg_config::Config::new(); + if is_loadable_extension() { + config.cargo_metadata(false); + } + config + } + + // Prints the necessary cargo link commands and returns the path to the header. + fn find_duckdb(_out_dir: &str) -> HeaderLocation { + println!("cargo:rerun-if-env-changed=DUCKDB_DOWNLOAD_LIB"); + if !is_loadable_extension() { + println!("cargo:rerun-if-env-changed=DUCKDB_INCLUDE_DIR"); + println!("cargo:rerun-if-env-changed=DUCKDB_LIB_DIR"); + println!("cargo:rerun-if-env-changed=DUCKDB_STATIC"); + if cfg!(feature = "vcpkg") && is_compiler("msvc") { + println!("cargo:rerun-if-env-changed=VCPKGRS_DYNAMIC"); + } + + // dependents can access `DEP_DUCKDB_LINK_TARGET` (`duckdb` being the + // `links=` value in our Cargo.toml) to get this value. This might be + // useful if you need to ensure whatever crypto library sqlcipher relies + // on is available, for example. + println!("cargo:link-target=duckdb"); + } + + if win_target() && cfg!(feature = "winduckdb") { + emit_link_lib("dylib=duckdb"); + return HeaderLocation::Wrapper; + } + // Allow users to specify where to find DuckDB. + if let Ok(dir) = env::var("DUCKDB_LIB_DIR") { + println!("cargo:rustc-env=LD_LIBRARY_PATH={dir}"); + // Try to use pkg-config to determine link commands + let pkgconfig_path = Path::new(&dir).join("pkgconfig"); + unsafe { env::set_var("PKG_CONFIG_PATH", pkgconfig_path) }; + + #[cfg(feature = "pkg-config")] + let lib_found = duckdb_pkg_config().probe("duckdb").is_ok(); + #[cfg(not(feature = "pkg-config"))] + let lib_found = false; + + if !lib_found { + // Otherwise just emit the bare minimum link commands. + emit_link_lib(link_directive()); + println!("cargo:rustc-link-search={dir}"); + } + + return HeaderLocation::from_env(Path::new(&dir)); + } + + if should_download_libduckdb() { + #[cfg(feature = "download")] + return download_libduckdb(_out_dir).unwrap_or_else(|err| panic!("Failed to download libduckdb: {err}")); + #[cfg(not(feature = "download"))] + panic!("DUCKDB_DOWNLOAD_LIB requires the libduckdb-sys/download feature"); + } + + if let Some(header) = try_vcpkg() { + return header; + } + + // See if pkg-config can do everything for us. + #[cfg(feature = "pkg-config")] + { + match duckdb_pkg_config().print_system_libs(false).probe("duckdb") { + Ok(mut lib) => { + if let Some(header) = lib.include_paths.pop() { + HeaderLocation::IncludeDir(header) + } else { + HeaderLocation::Wrapper + } + } + Err(_) => { + // No env var set and pkg-config couldn't help; just output the link-lib + // request and hope that the library exists on the system paths. We used to + // output /usr/lib explicitly, but that can introduce other linking problems; + // see https://github.com/rusqlite/rusqlite/issues/207. + emit_link_lib(link_directive()); + HeaderLocation::Wrapper + } + } + } + #[cfg(not(feature = "pkg-config"))] + { + // No pkg-config available; just output the link-lib request and hope + // that the library exists on the system paths. + emit_link_lib(link_directive()); + HeaderLocation::Wrapper + } + } + + fn try_vcpkg() -> Option { + #[cfg(feature = "vcpkg")] + if is_compiler("msvc") { + // See if vcpkg can find it. + if let Ok(mut lib) = vcpkg::Config::new().probe("duckdb") { + if let Some(header) = lib.include_paths.pop() { + return Some(HeaderLocation::IncludeDir(header)); + } + } + } + None + } + + fn should_download_libduckdb() -> bool { + env::var("DUCKDB_DOWNLOAD_LIB") + .map(|value| matches!(value.to_ascii_lowercase().as_str(), "1" | "true")) + .unwrap_or(false) + } + + #[cfg(feature = "download")] + fn download_libduckdb(out_dir: &str) -> Result> { + let target = env::var("TARGET")?; + let archive = LibduckdbArchive::for_target(&target) + .ok_or_else(|| format!("No pre-built libduckdb available for target '{target}'"))?; + + let version = duckdb_version_from_pkg_version(env!("CARGO_PKG_VERSION")); + + // Cache downloads in target/duckdb-download// so successive builds reuse them + let download_dir = workspace_download_dir(out_dir)?.join(&target).join(&version); + fs::create_dir_all(&download_dir)?; + + let archive_path = download_dir.join(archive.archive_name); + let lib_marker = download_dir.join(archive.dynamic_lib); + + if lib_marker.exists() { + println!("cargo:warning=Reusing libduckdb from {}", download_dir.display()); + } else { + let client = http_client()?; + let url = archive.download_url(&version); + ensure_libduckdb(&client, &url, &archive_path)?; + extract_libduckdb(&archive_path, &download_dir)?; + if !lib_marker.exists() { + return Err(format!( + "Downloaded archive did not contain expected library '{}'", + archive.dynamic_lib + ) + .into()); + } + } + + configure_link_search(&download_dir); + + copy_libduckdb(&download_dir, archive.dynamic_lib, out_dir)?; + + Ok(HeaderLocation::IncludeDir(download_dir)) + } + + #[cfg(feature = "download")] + fn configure_link_search(lib_dir: &Path) { + println!("cargo:rustc-link-search=native={}", lib_dir.display()); + emit_link_lib(link_directive()); + if !win_target() { + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_dir.display()); + } + } + + // Ensures the libduckdb archive exists: reuses an existing zip or + // downloads it into a temp file and atomically renames it into place. + #[cfg(feature = "download")] + fn ensure_libduckdb( + client: &reqwest::blocking::Client, + url: &str, + archive_path: &Path, + ) -> Result<(), Box> { + if archive_path.exists() { + println!("cargo:warning=libduckdb already present at {}", archive_path.display()); + return Ok(()); + } + let tmp_path = archive_path.with_extension("download"); + if let Some(parent) = archive_path.parent() { + fs::create_dir_all(parent)?; + } + let mut response = client.get(url).send()?.error_for_status()?; + let mut tmp_file = fs::File::create(&tmp_path)?; + io::copy(&mut response, &mut tmp_file)?; + fs::rename(&tmp_path, archive_path)?; + println!("cargo:warning=Downloaded libduckdb from {url}"); + Ok(()) + } + + #[cfg(feature = "download")] + fn extract_libduckdb(archive_path: &Path, destination: &Path) -> Result<(), Box> { + let file = fs::File::open(archive_path)?; + let mut archive = zip::ZipArchive::new(file)?; + archive.extract(destination)?; + println!("cargo:warning=Extracted libduckdb to {}", destination.display()); + Ok(()) + } + + // Copy libduckdb into target//deps so executables/tests can load it via + // the default Cargo rpath, just like when DUCKDB_LIB_DIR is set. + #[cfg(feature = "download")] + fn copy_libduckdb( + download_dir: &Path, + lib_filename: &str, + out_dir: &str, + ) -> Result<(), Box> { + let Some(deps_dir) = profile_deps_dir(out_dir) else { + return Err("Could not determine target/deps directory for runtime lib copy".into()); + }; + fs::create_dir_all(&deps_dir)?; + let source = download_dir.join(lib_filename); + let dest = deps_dir.join(lib_filename); + if dest.exists() { + fs::remove_file(&dest)?; + } + fs::copy(&source, &dest)?; + println!("cargo:warning=Copied libduckdb to {}", dest.display()); + Ok(()) + } + + // Finds target/ so downloads survive rebuilds. Respects CARGO_TARGET_DIR if set. + #[cfg(feature = "download")] + fn workspace_download_dir(out_dir: &str) -> Result> { + if let Ok(dir) = env::var("CARGO_TARGET_DIR") { + return Ok(PathBuf::from(dir).join("duckdb-download")); + } + let target_root = Path::new(out_dir) + .ancestors() + .find(|ancestor| { + ancestor + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name == "target") + }) + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from("target")); + Ok(target_root.join("duckdb-download")) + } + + #[cfg(feature = "download")] + fn duckdb_version_from_pkg_version(pkg_version: &str) -> String { + // duckdb-rs uses 1.MAJOR_MINOR_PATCH.x, e.g. DuckDB 1.5.0 => duckdb-rs 1.10500.x. + let encoded = pkg_version + .split('.') + .nth(1) + .expect("CARGO_PKG_VERSION should use the documented 1.MAJOR_MINOR_PATCH.x format") + .parse::() + .expect("CARGO_PKG_VERSION should encode the DuckDB version as an integer in its second component"); + let duckdb_major = encoded / 10_000; + let duckdb_minor = (encoded / 100) % 100; + let duckdb_patch = encoded % 100; + format!("{duckdb_major}.{duckdb_minor}.{duckdb_patch}") + } + + // Mirrors Cargo's actual target///deps (or target//deps) + // layout by deriving it from OUT_DIR. + #[cfg(feature = "download")] + fn profile_deps_dir(out_dir: &str) -> Option { + let build_dir = Path::new(out_dir).ancestors().find(|ancestor| { + ancestor + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name == "build") + })?; + let profile_dir = build_dir.parent()?; + Some(profile_dir.join("deps")) + } + + #[cfg(feature = "download")] + struct LibduckdbArchive { + archive_name: &'static str, + dynamic_lib: &'static str, + } + + #[cfg(feature = "download")] + impl LibduckdbArchive { + fn for_target(target: &str) -> Option { + match target { + t if t.ends_with("apple-darwin") => Some(Self { + archive_name: "libduckdb-osx-universal.zip", + dynamic_lib: "libduckdb.dylib", + }), + "x86_64-unknown-linux-gnu" => Some(Self { + archive_name: "libduckdb-linux-amd64.zip", + dynamic_lib: "libduckdb.so", + }), + "aarch64-unknown-linux-gnu" => Some(Self { + archive_name: "libduckdb-linux-arm64.zip", + dynamic_lib: "libduckdb.so", + }), + "x86_64-pc-windows-msvc" => Some(Self { + archive_name: "libduckdb-windows-amd64.zip", + dynamic_lib: "duckdb.dll", + }), + "aarch64-pc-windows-msvc" => Some(Self { + archive_name: "libduckdb-windows-arm64.zip", + dynamic_lib: "duckdb.dll", + }), + _ => None, + } + } + + fn download_url(&self, version: &str) -> String { + format!( + "https://github.com/duckdb/duckdb/releases/download/v{version}/{}", + self.archive_name + ) + } + } + + #[cfg(feature = "download")] + fn http_client() -> Result { + let timeout = env::var("CARGO_HTTP_TIMEOUT") + .or_else(|_| env::var("HTTP_TIMEOUT")) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(90); + reqwest::blocking::Client::builder() + .timeout(std::time::Duration::from_secs(timeout)) + .build() + } +} + +#[cfg(feature = "buildtime_bindgen")] +mod bindings { + use super::{HeaderLocation, is_loadable_extension}; + + use std::{fs::OpenOptions, io::Write, path::Path}; + + #[cfg(feature = "loadable-extension")] + fn extract_method(ty: &syn::Type) -> Option<&syn::TypeBareFn> { + let syn::Type::Path(type_path) = ty else { + return None; + }; + let segment = type_path.path.segments.last()?; + let syn::PathArguments::AngleBracketed(args) = &segment.arguments else { + return None; + }; + let Some(syn::GenericArgument::Type(ty)) = args.args.first() else { + return None; + }; + let syn::Type::BareFn(method) = ty else { + return None; + }; + Some(method) + } + + #[cfg(feature = "loadable-extension")] + fn generate_functions(mut output: String) -> String { + // (1) parse sqlite3_api_routines fields from bindgen output + let ast: syn::File = syn::parse_str(&output).expect("could not parse bindgen output"); + let duckdb_ext_api_v1: syn::ItemStruct = ast + .items + .into_iter() + .find_map(|i| { + if let syn::Item::Struct(s) = i { + if s.ident == "duckdb_ext_api_v1" { Some(s) } else { None } + } else { + None + } + }) + .expect("could not find duckdb_ext_api_v1"); + + let p_api = quote::format_ident!("p_api"); + let mut stores = Vec::new(); + + for field in duckdb_ext_api_v1.fields { + let ident = field.ident.expect("unnamed field"); + let span = ident.span(); + let function_name = ident.to_string(); + let ptr_name = syn::Ident::new(format!("__{}", function_name.to_uppercase()).as_ref(), span); + + // Create syntax name + let duckdb_fn_name = syn::Ident::new(&function_name, span); + + let method = extract_method(&field.ty).unwrap_or_else(|| panic!("unexpected type for {function_name}")); + + let arg_names: syn::punctuated::Punctuated<&syn::Ident, syn::token::Comma> = + method.inputs.iter().map(|i| &i.name.as_ref().unwrap().0).collect(); + + let args = &method.inputs; + + let ty = &method.output; + + let tokens = quote::quote! { + static #ptr_name: ::std::sync::atomic::AtomicPtr<()> = ::std::sync::atomic::AtomicPtr::new(::std::ptr::null_mut()); + pub unsafe fn #duckdb_fn_name(#args) #ty { + let function_ptr = #ptr_name.load(::std::sync::atomic::Ordering::Acquire); + assert!(!function_ptr.is_null(), "DuckDB API not initialized or DuckDB feature omitted"); + let fun: unsafe extern "C" fn(#args) #ty = ::std::mem::transmute(function_ptr); + (fun)(#arg_names) + } + }; + + output.push_str(&prettyplease::unparse( + &syn::parse2(tokens).expect("could not parse quote output"), + )); + + output.push('\n'); + + stores.push(quote::quote! { + if let Some(fun) = (*#p_api).#ident { + #ptr_name.store( + fun as usize as *mut (), + ::std::sync::atomic::Ordering::Release, + ); + } + }); + } + + // (3) generate rust code similar to DUCKDB_EXTENSION_API_INIT macro + let tokens = quote::quote! { + /// Like DUCKDB_EXTENSION_API_INIT macro + pub unsafe fn duckdb_rs_extension_api_init(info: duckdb_extension_info, access: *const duckdb_extension_access, version: &str) -> ::std::result::Result { + let version_c_string = std::ffi::CString::new(version).unwrap(); + let #p_api = (*access).get_api.unwrap()(info, version_c_string.as_ptr()) as *const duckdb_ext_api_v1; + if #p_api.is_null() { + // get_api can return a nullptr when the version is not matched. In this case, we don't need to set + // an error, but can instead just stop the initialization process and let duckdb handle things + return Ok(false); + } + #(#stores)* + Ok(true) + } + }; + output.push_str(&prettyplease::unparse( + &syn::parse2(tokens).expect("could not parse quote output"), + )); + output.push('\n'); + output + } + + pub fn write_to_out_dir(header: HeaderLocation, out_path: &Path) { + let header = header.header_path().to_string_lossy().into_owned(); + let mut output = Vec::new(); + let mut builder = bindgen::builder(); + + if is_loadable_extension() { + builder = builder.ignore_functions(); + } + + // ONLY generate bindings for symbols containing "duckdb" in their name + // and for the type `idx_t` + // We have to pass DDUCKDB_EXTENSION_API_VERSION_UNSTABLE for now, + // until we figure out how to feature gate the generated API + builder + .trust_clang_mangling(false) + .header(header.clone()) + .allowlist_item(r#"(\w*duckdb\w*)"#) + .allowlist_type("idx_t") + // Use the concrete ABI layouts from src/arrow_c_data.rs. + .blocklist_type("ArrowArray") + .blocklist_type("ArrowSchema") + .layout_tests(false) // causes problems on WASM builds + .clang_arg("-DDUCKDB_EXTENSION_API_VERSION_UNSTABLE") + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) + .generate() + .unwrap_or_else(|_| panic!("could not run bindgen on header {header}")) + .write(Box::new(&mut output)) + .expect("could not write output of bindgen"); + + let output = String::from_utf8(output).expect("bindgen output was not UTF-8?!"); + + #[cfg(feature = "loadable-extension")] + let output = generate_functions(output); + + let mut file = OpenOptions::new() + .write(true) + .truncate(true) + .create(true) + .open(out_path) + .unwrap_or_else(|_| panic!("Could not write to {out_path:?}")); + + file.write_all(output.as_bytes()) + .unwrap_or_else(|_| panic!("Could not write to {out_path:?}")); + } +} diff --git a/rust_hft/native/libduckdb-sys/build_bundled_cc.rs b/rust_hft/native/libduckdb-sys/build_bundled_cc.rs new file mode 100644 index 000000000..7c9e0e22a --- /dev/null +++ b/rust_hft/native/libduckdb-sys/build_bundled_cc.rs @@ -0,0 +1,147 @@ +use crate::{is_compiler, link_windows_system_libs, win_target, write_bindings}; +use std::{ + collections::{HashMap, HashSet}, + path::Path, +}; + +#[derive(serde::Deserialize)] +struct Sources { + cpp_files: HashSet, + include_dirs: HashSet, +} + +#[derive(serde::Deserialize)] +struct Manifest { + base: Sources, + extensions: HashMap, +} + +fn add_extension( + manifest: &Manifest, + extension: &str, + cpp_files: &mut HashSet, + include_dirs: &mut HashSet, +) { + let sources = manifest.extensions.get(extension).unwrap_or_else(|| { + let mut available = manifest.extensions.keys().cloned().collect::>(); + available.sort(); + panic!( + "extension `{extension}` is missing from duckdb manifest; available extensions: {}", + available.join(", ") + ); + }); + cpp_files.extend(sources.cpp_files.clone()); + include_dirs.extend(sources.include_dirs.clone()); +} + +fn extension_enabled(extension: &str) -> bool { + // Review build_bundled_cmake::enabled_extensions when changing this gate; + // the backend mechanisms and supported extension sets intentionally differ. + extension == "core_functions" + || (extension == "parquet" && cfg!(feature = "parquet")) + || (extension == "json" && cfg!(feature = "json")) +} + +fn untar_archive(out_dir: &str) { + let path = "duckdb.tar.gz"; + + let tar_gz = std::fs::File::open(path).expect("archive file"); + let tar = flate2::read::GzDecoder::new(tar_gz); + let mut archive = tar::Archive::new(tar); + // These archives are generated for reproducibility, not source mtimes. + // Skipping mtime restoration also avoids Windows SetFileTime failures on + // filesystems such as FAT32, which cannot represent Unix epoch mtimes. + archive.set_preserve_mtime(false); + archive.unpack(out_dir).expect("archive"); +} + +pub fn main(out_dir: &str, out_path: &Path) { + untar_archive(out_dir); + + let include_path = Path::new(out_dir).join("duckdb/src/include"); + write_bindings(&include_path, out_path); + + // Publish the include directory so downstream crates that compile + // their own C/C++ code (e.g. extension shims that #include + // "duckdb.hpp") can pick it up from `DEP_DUCKDB_INCLUDE` without + // having to glob the target tree. Requires `links = "duckdb"` in + // the package manifest. + println!("cargo:include={}", include_path.display()); + + let manifest_file = std::fs::File::open(format!("{out_dir}/duckdb/manifest.json")).expect("manifest file"); + let manifest: Manifest = serde_json::from_reader(manifest_file).expect("reading manifest file"); + + let mut cpp_files = HashSet::new(); + let mut include_dirs = HashSet::new(); + let mut extensions = manifest + .extensions + .keys() + .filter(|name| extension_enabled(name)) + .cloned() + .collect::>(); + extensions.sort(); + + cpp_files.extend(manifest.base.cpp_files.clone()); + include_dirs.extend(manifest.base.include_dirs.iter().cloned()); + + let mut cfg = cc::Build::new(); + + for extension in &extensions { + add_extension(&manifest, extension, &mut cpp_files, &mut include_dirs); + cfg.define( + &format!("DUCKDB_EXTENSION_{}_LINKED", extension.to_uppercase()), + Some("1"), + ); + } + cfg.define("DUCKDB_EXTENSION_AUTOINSTALL_DEFAULT", "1"); + cfg.define("DUCKDB_EXTENSION_AUTOLOAD_DEFAULT", "1"); + + println!("cargo:rerun-if-changed=duckdb.tar.gz"); + + cfg.include("duckdb"); + cfg.includes(include_dirs.iter().map(|dir| format!("{out_dir}/duckdb/{dir}"))); + + let mut cpp_files_vec: Vec = cpp_files.into_iter().collect(); + cpp_files_vec.sort(); + for f in cpp_files_vec.into_iter().map(|file| format!("{out_dir}/{file}")) { + cfg.file(f); + } + + cfg.cpp(true) + .flag_if_supported("-std=c++11") + .flag_if_supported("/utf-8") + .flag_if_supported("/bigobj") + .warnings(false) + .flag_if_supported("-w"); + + // Enable C++ exceptions on MSVC: without /EHsc, exception handling is off, the + // C API's try/catch blocks are inert, and a thrown exception (e.g. invalid-UTF-8 + // CSV read) aborts with STATUS_STACK_BUFFER_OVERRUN (#774). Hard flag, not + // flag_if_supported, so a spurious probe miss can't silently drop it. gcc/clang + // enable exceptions by default and reject /EHsc, so gate on MSVC. Mirrors + // build_bundled_cmake.rs. + if win_target() && is_compiler("msvc") { + cfg.flag("/EHsc"); + } + + let is_debug = match std::env::var("DEBUG") { + Ok(v) => v != "false" && v != "0", + Err(_) => false, + }; + if !is_debug { + cfg.define("NDEBUG", None); + } + + if win_target() { + cfg.define("DUCKDB_BUILD_LIBRARY", None); + } + cfg.compile("duckdb"); + + // `cc` does not link DuckDB's Windows system libs automatically (e.g. unresolved + // `RmStartSession` from the Restart Manager). See link_windows_system_libs. + if win_target() { + link_windows_system_libs(); + } + + println!("cargo:lib_dir={out_dir}"); +} diff --git a/rust_hft/native/libduckdb-sys/build_bundled_cmake.rs b/rust_hft/native/libduckdb-sys/build_bundled_cmake.rs new file mode 100644 index 000000000..49194d0ec --- /dev/null +++ b/rust_hft/native/libduckdb-sys/build_bundled_cmake.rs @@ -0,0 +1,496 @@ +use crate::{is_compiler, link_windows_system_libs, win_target, write_bindings}; +use std::{ + collections::BTreeSet, + env::{self, VarError}, + fs, io, + path::{Path, PathBuf}, +}; + +struct Generator { + name: Option, + out_dir: PathBuf, + make_program: Option, +} + +pub fn main(out_dir: &str, out_path: &Path) { + let source_dir = Path::new("duckdb-sources"); + let cmake_lists = source_dir.join("CMakeLists.txt"); + if !cmake_lists.exists() { + panic!( + "`bundled-cmake` requires a duckdb-rs checkout with DuckDB sources at {}", + cmake_lists.display() + ); + } + + println!("cargo:rerun-if-changed={}", source_dir.display()); + println!("cargo:rerun-if-env-changed=DUCKDB_EXTENSION_CONFIGS"); + println!("cargo:rerun-if-env-changed=DUCKDB_CMAKE_BUILD_TYPE"); + println!("cargo:rerun-if-env-changed=DUCKDB_DISABLE_UNITY"); + println!("cargo:rerun-if-env-changed=DUCKDB_DISABLE_JEMALLOC"); + println!("cargo:rerun-if-env-changed=DUCKDB_DISABLE_EXTENSION_LOAD"); + println!("cargo:rerun-if-env-changed=DISABLE_EXTENSION_LOAD"); + println!("cargo:rerun-if-env-changed=CMAKE_BUILD_TYPE"); + println!("cargo:rerun-if-env-changed=CMAKE_GENERATOR"); + println!("cargo:rerun-if-env-changed=CMAKE_C_COMPILER_LAUNCHER"); + println!("cargo:rerun-if-env-changed=CMAKE_CXX_COMPILER_LAUNCHER"); + println!("cargo:rerun-if-env-changed=MACOSX_DEPLOYMENT_TARGET"); + + let include_path = source_dir.join("src/include").canonicalize().unwrap(); + write_bindings(&include_path, out_path); + + // Publish the include directory so downstream crates that compile + // their own C/C++ code can read it from `DEP_DUCKDB_INCLUDE`. + // See the matching emission in build_bundled_cc.rs. + println!("cargo:include={}", include_path.display()); + + if let Some(configs) = env_var("DUCKDB_EXTENSION_CONFIGS") { + if !configs.trim().is_empty() { + panic!( + "DUCKDB_EXTENSION_CONFIGS is not yet supported by bundled-cmake because additional static extension libraries are not auto-linked" + ); + } + } + + let cmake_build_type = cmake_build_type(); + let generator = select_generator(out_dir); + let mut config = cmake::Config::new(source_dir); + config.out_dir(&generator.out_dir); + if let Some(generator_name) = generator.name.as_deref() { + config.generator(generator_name); + } + if let Some(make_program) = generator.make_program.as_deref() { + config.define("CMAKE_MAKE_PROGRAM", make_program); + } + configure_macos_deployment_target(&mut config); + config + .profile(&cmake_build_type) + .define("BUILD_UNITTESTS", "0") + .define("BUILD_SHELL", "0") + .define("CMAKE_INSTALL_LIBDIR", "lib") + .define("CMAKE_C_FLAGS_INIT", warning_suppression_flag()) + .define("CMAKE_CXX_FLAGS_INIT", warning_suppression_flag()); + + // Re-enable C++ exceptions on MSVC. The cmake crate makes its `cxxflag`s the + // source of -DCMAKE_CXX_FLAGS, replacing CMake's default (which includes /EHsc), + // so exception handling ends up effectively off: a thrown exception (e.g. + // invalid-UTF-8 CSV read) then aborts with STATUS_STACK_BUFFER_OVERRUN (#774). + // + // Don't `config.define("CMAKE_CXX_FLAGS", ..)` here: it overrides the cxxflag + // above and silently drops /EHsc, reintroducing #774. + // Mirrors build_bundled_cc.rs; gcc/clang enable exceptions by default. + if win_target() && is_compiler("msvc") { + config.cxxflag("/EHsc"); + } + + // Unity builds (DuckDB's default) combine .cpp files into fewer translation + // units and compile significantly faster. Allow opting out for debugging. + // Always set explicitly so the CMake cache doesn't keep a stale value. + let disable_unity = env_flag("DUCKDB_DISABLE_UNITY"); + config.define("DISABLE_UNITY", if disable_unity { "1" } else { "0" }); + + // Always set explicitly so the CMake cache does not keep a stale value when + // the environment variable changes between builds. + let disable_extension_load = env_flag_aliases(&["DUCKDB_DISABLE_EXTENSION_LOAD", "DISABLE_EXTENSION_LOAD"]); + config.define("DISABLE_EXTENSION_LOAD", if disable_extension_load { "1" } else { "0" }); + + // Forward compiler launcher for sccache/ccache integration. + for var in ["CMAKE_C_COMPILER_LAUNCHER", "CMAKE_CXX_COMPILER_LAUNCHER"] { + if let Some(launcher) = env_var(var) { + config.define(var, &launcher); + } + } + + let enabled_extensions = enabled_extensions(); + if !enabled_extensions.is_empty() { + config.define("BUILD_EXTENSIONS", enabled_extensions.join(";")); + } + + // Always set explicitly so old CMake caches do not keep stale allocator + // state. By default this enables jemalloc on DuckDB's supported 64-bit, + // non-musl Linux targets and remains a no-op elsewhere. The env var is an + // escape hatch back to DuckDB's standard allocator. + let disable_jemalloc = env_flag("DUCKDB_DISABLE_JEMALLOC"); + config.define("SKIP_EXTENSIONS", ""); + config.define("ENABLE_JEMALLOC", if disable_jemalloc { "OFF" } else { "ON" }); + + // Upstream CMake defaults these to OFF, but duckdb-rs `bundled` has historically + // shipped with both enabled. Keep `bundled-cmake` aligned with `bundled` unless + // extension loading is compiled out entirely. + config + .define( + "ENABLE_EXTENSION_AUTOLOADING", + if disable_extension_load { "0" } else { "1" }, + ) + .define( + "ENABLE_EXTENSION_AUTOINSTALL", + if disable_extension_load { "0" } else { "1" }, + ); + + // Windows MAX_PATH caveat: this build emits very deep object paths (Ninja ignores + // CMAKE_OBJECT_PATH_MAX), so a deep OUT_DIR can exceed 260 chars and fail late with + // `C1083: Cannot open compiler generated file: ''`. The build script can't move + // OUT_DIR — shorten the path (short CARGO_TARGET_DIR, drop `--target`, or check out + // nearer the drive root). + let dst = config.build(); + let lib_dir = dst.join("lib"); + validate_extension_libraries(&lib_dir, &cmake_build_type, &enabled_extensions); + println!("cargo:rustc-link-search=native={}", lib_dir.display()); + // Emit in dependents-before-dependencies order for single-pass linkers: + // loader → extensions → duckdb_static (which satisfies all core symbols). + link_static_library(&lib_dir, &cmake_build_type, "duckdb_generated_extension_loader"); + link_static_library(&lib_dir, &cmake_build_type, "core_functions_extension"); + for extension in enabled_extensions { + link_static_library(&lib_dir, &cmake_build_type, &format!("{extension}_extension")); + } + link_static_library(&lib_dir, &cmake_build_type, "duckdb_static"); + link_system_libs(); + println!("cargo:lib_dir={}", lib_dir.display()); +} + +fn cmake_build_type() -> String { + for var in ["DUCKDB_CMAKE_BUILD_TYPE", "CMAKE_BUILD_TYPE"] { + if let Some(value) = env_var(var).filter(|v| !v.trim().is_empty()) { + let value = validate_cmake_build_type(&value); + cargo_warning(&format!("bundled-cmake build type: {value} (from {var})")); + return value; + } + } + cargo_warning("bundled-cmake build type: Release (default)"); + "Release".to_owned() +} + +fn validate_cmake_build_type(value: &str) -> String { + match value { + "Debug" | "Release" | "RelWithDebInfo" | "MinSizeRel" => value.to_owned(), + _ => { + panic!("unsupported CMake build type `{value}`; expected one of Debug, Release, RelWithDebInfo, MinSizeRel") + } + } +} + +fn select_generator(out_dir: &str) -> Generator { + let out_dir = PathBuf::from(out_dir); + if let Some(generator) = env_var("CMAKE_GENERATOR").filter(|value| !value.trim().is_empty()) { + let needs_ninja = generator.contains("Ninja"); + let (make_program, probe_error) = if needs_ninja { + find_program(&["ninja", "ninja-build"]) + } else { + (None, None) + }; + if needs_ninja && make_program.is_none() { + let suffix = probe_error + .map(|error| format!(" (probe error: {error})")) + .unwrap_or_default(); + panic!( + "CMAKE_GENERATOR={generator} requires `ninja` or `ninja-build` on PATH, but neither was found{suffix}" + ); + } + cargo_warning(&format!("bundled-cmake generator: {generator} (from CMAKE_GENERATOR)")); + return Generator { + out_dir: out_dir.join(format!("cmake-{}", sanitize_path_component(&generator))), + name: Some(generator), + make_program, + }; + } + + let (ninja_program, probe_error) = find_program(&["ninja", "ninja-build"]); + if let Some(ninja) = ninja_program { + cargo_warning(&format!("bundled-cmake generator: Ninja (autodetected via {ninja})")); + Generator { + out_dir: out_dir.join("cmake-ninja"), + name: Some("Ninja".to_string()), + make_program: Some(ninja), + } + } else { + if let Some(probe_error) = probe_error { + cargo_warning(&format!( + "failed to probe for bundled-cmake generator detection (`ninja`/`ninja-build`): {probe_error}" + )); + } + cargo_warning("bundled-cmake generator: default CMake generator (ninja not detected)"); + Generator { + out_dir: out_dir.join("cmake-default"), + name: None, + make_program: None, + } + } +} + +fn warning_suppression_flag() -> &'static str { + if win_target() && is_compiler("msvc") { + "/w" + } else { + "-w" + } +} + +fn configure_macos_deployment_target(config: &mut cmake::Config) { + if env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("macos") { + return; + } + + let target_arch_env = + env::var("CARGO_CFG_TARGET_ARCH").expect("Cargo should set CARGO_CFG_TARGET_ARCH for macOS builds"); + let (target_arch, default_deployment_target) = match target_arch_env.as_str() { + "aarch64" => ("arm64", "11.0"), + "x86_64" => ("x86_64", "10.15"), + other => { + panic!("bundled-cmake: unsupported macOS CARGO_CFG_TARGET_ARCH `{other}`; expected `aarch64` or `x86_64`") + } + }; + + let deployment_target = env_var("MACOSX_DEPLOYMENT_TARGET") + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| default_deployment_target.to_string()); + cargo_warning(&format!( + "bundled-cmake macOS deployment target: {deployment_target} ({target_arch})" + )); + config.define("CMAKE_OSX_DEPLOYMENT_TARGET", &deployment_target); + config.define("CMAKE_OSX_ARCHITECTURES", target_arch); +} + +fn find_program(candidates: &[&str]) -> (Option, Option) { + let mut last_error = None; + for candidate in candidates { + match which::which(candidate) { + Ok(path) => return (Some(path.display().to_string()), None), + Err(which::Error::CannotFindBinaryPath) => {} + Err(err) => { + last_error = Some(err); + } + } + } + (None, last_error) +} + +fn env_var(name: &str) -> Option { + match env::var(name) { + Ok(value) => Some(value), + Err(VarError::NotPresent) => None, + Err(VarError::NotUnicode(_)) => { + panic!("bundled-cmake expects a valid UTF-8 value for environment variable `{name}`") + } + } +} + +fn parse_env_flag(name: &str) -> Option { + match env_var(name).as_deref() { + Some("1" | "true" | "on" | "yes") => Some(true), + Some("0" | "false" | "off" | "no") => Some(false), + Some(other) => { + cargo_warning(&format!( + "Ignoring unsupported {name} value {other:?}; expected true/false or 1/0, yes/no, on/off" + )); + None + } + None => None, + } +} + +fn env_flag(name: &str) -> bool { + parse_env_flag(name).unwrap_or(false) +} + +fn env_flag_aliases(names: &[&str]) -> bool { + let mut values = Vec::new(); + for name in names { + if let Some(value) = parse_env_flag(name) { + values.push((*name, value)); + } + } + let Some((selected_name, selected_value)) = values.first().copied() else { + return false; + }; + if values.iter().any(|(_, value)| *value != selected_value) { + let sources = values + .iter() + .map(|(name, value)| format!("{name}={}", if *value { 1 } else { 0 })) + .collect::>() + .join(", "); + cargo_warning(&format!( + "Conflicting values for equivalent environment variables ({sources}); using {selected_name}" + )); + } + selected_value +} + +fn sanitize_path_component(input: &str) -> String { + input + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' { + ch + } else { + '_' + } + }) + .collect() +} + +fn validate_extension_libraries(lib_dir: &Path, cmake_build_type: &str, enabled_extensions: &[&str]) { + // Missing extensions are already caught by link_static_library's panic; + // this check only guards against CMake producing extensions we did not + // ask for (e.g. upstream extension-config drift). + let actual = list_static_extension_libraries(lib_dir, cmake_build_type).unwrap_or_else(|err| { + panic!( + "failed to inspect bundled-cmake extension libraries in {}: {err}", + lib_dir.display() + ) + }); + let mut expected: BTreeSet = enabled_extensions + .iter() + .map(|ext| format!("{ext}_extension")) + .collect(); + // CMake always builds this base extension through extension_config.cmake. + expected.insert("core_functions_extension".to_string()); + + let unexpected: Vec = actual.difference(&expected).cloned().collect(); + if !unexpected.is_empty() { + panic!( + "bundled-cmake produced unexpected static extension libraries: {}", + unexpected.join(", ") + ); + } +} + +fn link_static_library(lib_dir: &Path, cmake_build_type: &str, name: &str) { + let Some(library_path) = resolve_static_library(lib_dir, cmake_build_type, name) else { + let filename = static_library_filename(name); + let searched = candidate_library_dirs(lib_dir, cmake_build_type) + .into_iter() + .map(|dir| dir.join(&filename).display().to_string()) + .collect::>() + .join(", "); + panic!("expected bundled-cmake to produce static library `{name}`; looked in [{searched}]"); + }; + if let Some(parent) = library_path.parent() { + if parent != lib_dir { + println!("cargo:rustc-link-search=native={}", parent.display()); + } + } + println!("cargo:rustc-link-lib=static={name}"); +} + +fn static_library_filename(name: &str) -> String { + if win_target() { + format!("{name}.lib") + } else { + format!("lib{name}.a") + } +} + +fn list_static_extension_libraries(lib_dir: &Path, cmake_build_type: &str) -> io::Result> { + let mut extension_libraries = BTreeSet::new(); + for candidate_dir in candidate_library_dirs(lib_dir, cmake_build_type) { + for entry in fs::read_dir(&candidate_dir)? { + let entry = entry?; + let file_type = entry.file_type()?; + if !file_type.is_file() { + continue; + } + let file_name = entry.file_name(); + let Some(file_name) = file_name.to_str() else { + cargo_warning(&format!( + "bundled-cmake ignored non-UTF-8 library filename in {}: {}", + candidate_dir.display(), + entry.path().display() + )); + continue; + }; + let Some(name) = normalized_static_library_name(file_name) else { + continue; + }; + if name.ends_with("_extension") && !name.ends_with("_loadable_extension") { + extension_libraries.insert(name); + } + } + } + Ok(extension_libraries) +} + +fn normalized_static_library_name(filename: &str) -> Option { + if let Some(name) = filename.strip_suffix(".lib") { + return Some(name.to_string()); + } + let name = filename.strip_prefix("lib")?.strip_suffix(".a")?; + Some(name.to_string()) +} + +fn resolve_static_library(lib_dir: &Path, cmake_build_type: &str, name: &str) -> Option { + let filename = static_library_filename(name); + candidate_library_dirs(lib_dir, cmake_build_type) + .into_iter() + .map(|dir| dir.join(&filename)) + .find(|path| path.exists()) +} + +fn candidate_library_dirs(lib_dir: &Path, cmake_build_type: &str) -> Vec { + // Single-config generators (Ninja, Make) write directly into lib/. + // Multi-config generators (MSVC, Xcode) write into lib//. + // Restricting to these two avoids picking up stale extension libraries + // from a previous build that used a different CMAKE_BUILD_TYPE. + let mut candidates = vec![lib_dir.to_path_buf()]; + let build_type_dir = lib_dir.join(cmake_build_type); + if build_type_dir.is_dir() { + candidates.push(build_type_dir); + } + candidates +} + +fn cargo_warning(message: &str) { + println!("cargo:warning={message}"); +} + +fn enabled_extensions() -> Vec<&'static str> { + // Match DuckDB's upstream CMake defaults for quasi-core extensions rather + // than trying to reproduce the cc backend's finer-grained selection. + // Review build_bundled_cc::extension_enabled when changing this list; the + // backend mechanisms and supported extension sets intentionally differ. + let mut extensions = vec!["parquet"]; + if cfg!(feature = "json") { + extensions.push("json"); + } + if cfg!(feature = "autocomplete") { + extensions.push("autocomplete"); + } + if cfg!(feature = "icu") { + extensions.push("icu"); + } + if cfg!(feature = "tpcds") { + extensions.push("tpcds"); + } + if cfg!(feature = "tpch") { + extensions.push("tpch"); + } + extensions +} + +fn link_system_libs() { + // Keep the per-OS lists below in sync with DuckDB's current linker requirements in + // duckdb-sources/src/CMakeLists.txt. If upstream drifts after a submodule bump, + // update every branch here alongside it. + let target_os = env::var("CARGO_CFG_TARGET_OS").expect("Cargo should set CARGO_CFG_TARGET_OS"); + + match target_os.as_str() { + "linux" => { + let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default(); + if target_env != "gnu" { + panic!( + "bundled-cmake on linux is currently only supported with the `gnu` target env (libstdc++); got `{target_env}`" + ); + } + println!("cargo:rustc-link-lib=dylib=dl"); + println!("cargo:rustc-link-lib=dylib=pthread"); + println!("cargo:rustc-link-lib=dylib=stdc++"); + } + "macos" => { + println!("cargo:rustc-link-lib=dylib=c++"); + } + "windows" => link_windows_system_libs(), + other => { + panic!( + "bundled-cmake is currently supported only on Linux, macOS, and Windows; unsupported target OS `{other}`" + ) + } + } +} diff --git a/rust_hft/native/libduckdb-sys/duckdb.tar.gz b/rust_hft/native/libduckdb-sys/duckdb.tar.gz new file mode 100644 index 000000000..7ce59da44 Binary files /dev/null and b/rust_hft/native/libduckdb-sys/duckdb.tar.gz differ diff --git a/rust_hft/native/libduckdb-sys/src/arrow_c_data.rs b/rust_hft/native/libduckdb-sys/src/arrow_c_data.rs new file mode 100644 index 000000000..d3cee1661 --- /dev/null +++ b/rust_hft/native/libduckdb-sys/src/arrow_c_data.rs @@ -0,0 +1,134 @@ +//! Arrow C Data Interface layouts used by DuckDB's Arrow conversion APIs. +//! +//! DuckDB's public C header forward-declares these structs. The conversion +//! APIs need concrete caller-allocated layouts, so `libduckdb-sys` defines the +//! ABI records directly without taking a dependency on arrow-rs. +//! +//! Specification: + +use std::{ + ffi::{c_char, c_void}, + ptr, +}; + +/// Arrow C Data Interface array. +#[repr(C)] +#[derive(Debug)] +pub struct ArrowArray { + pub length: i64, + pub null_count: i64, + pub offset: i64, + pub n_buffers: i64, + pub n_children: i64, + pub buffers: *mut *const c_void, + pub children: *mut *mut ArrowArray, + pub dictionary: *mut ArrowArray, + pub release: Option, + pub private_data: *mut c_void, +} + +impl ArrowArray { + /// Creates a null-release placeholder for a producer to fill. + pub const fn empty() -> Self { + Self { + length: 0, + null_count: 0, + offset: 0, + n_buffers: 0, + n_children: 0, + buffers: ptr::null_mut(), + children: ptr::null_mut(), + dictionary: ptr::null_mut(), + release: None, + private_data: ptr::null_mut(), + } + } +} + +/// Arrow C Data Interface schema. +#[repr(C)] +#[derive(Debug)] +pub struct ArrowSchema { + pub format: *const c_char, + pub name: *const c_char, + pub metadata: *const c_char, + pub flags: i64, + pub n_children: i64, + pub children: *mut *mut ArrowSchema, + pub dictionary: *mut ArrowSchema, + pub release: Option, + pub private_data: *mut c_void, +} + +impl ArrowSchema { + /// Creates a null-release placeholder for a producer to fill. + pub const fn empty() -> Self { + Self { + format: ptr::null(), + name: ptr::null(), + metadata: ptr::null(), + flags: 0, + n_children: 0, + children: ptr::null_mut(), + dictionary: ptr::null_mut(), + release: None, + private_data: ptr::null_mut(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::mem::{align_of, offset_of, size_of}; + + use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; + + macro_rules! assert_offsets { + ($left:ty, $right:ty, $($field:ident),+ $(,)?) => { + $( + assert_eq!( + offset_of!($left, $field), + offset_of!($right, $field), + concat!("field offset differs for ", stringify!($field)) + ); + )+ + }; + } + + #[test] + fn matches_arrow_rs_layouts() { + assert_eq!(size_of::(), size_of::()); + assert_eq!(align_of::(), align_of::()); + assert_offsets!( + ArrowArray, + FFI_ArrowArray, + length, + null_count, + offset, + n_buffers, + n_children, + buffers, + children, + dictionary, + release, + private_data, + ); + + assert_eq!(size_of::(), size_of::()); + assert_eq!(align_of::(), align_of::()); + assert_offsets!( + ArrowSchema, + FFI_ArrowSchema, + format, + name, + metadata, + flags, + n_children, + children, + dictionary, + release, + private_data, + ); + } +} diff --git a/rust_hft/native/libduckdb-sys/src/bindgen_bundled_version.rs b/rust_hft/native/libduckdb-sys/src/bindgen_bundled_version.rs new file mode 100644 index 000000000..d04184e77 --- /dev/null +++ b/rust_hft/native/libduckdb-sys/src/bindgen_bundled_version.rs @@ -0,0 +1,3579 @@ +/* automatically generated by rust-bindgen 0.72.1 */ + +pub const DUCKDB_TYPE_DUCKDB_TYPE_INVALID: DUCKDB_TYPE = 0; +pub const DUCKDB_TYPE_DUCKDB_TYPE_BOOLEAN: DUCKDB_TYPE = 1; +pub const DUCKDB_TYPE_DUCKDB_TYPE_TINYINT: DUCKDB_TYPE = 2; +pub const DUCKDB_TYPE_DUCKDB_TYPE_SMALLINT: DUCKDB_TYPE = 3; +pub const DUCKDB_TYPE_DUCKDB_TYPE_INTEGER: DUCKDB_TYPE = 4; +pub const DUCKDB_TYPE_DUCKDB_TYPE_BIGINT: DUCKDB_TYPE = 5; +pub const DUCKDB_TYPE_DUCKDB_TYPE_UTINYINT: DUCKDB_TYPE = 6; +pub const DUCKDB_TYPE_DUCKDB_TYPE_USMALLINT: DUCKDB_TYPE = 7; +pub const DUCKDB_TYPE_DUCKDB_TYPE_UINTEGER: DUCKDB_TYPE = 8; +pub const DUCKDB_TYPE_DUCKDB_TYPE_UBIGINT: DUCKDB_TYPE = 9; +pub const DUCKDB_TYPE_DUCKDB_TYPE_FLOAT: DUCKDB_TYPE = 10; +pub const DUCKDB_TYPE_DUCKDB_TYPE_DOUBLE: DUCKDB_TYPE = 11; +pub const DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP: DUCKDB_TYPE = 12; +pub const DUCKDB_TYPE_DUCKDB_TYPE_DATE: DUCKDB_TYPE = 13; +pub const DUCKDB_TYPE_DUCKDB_TYPE_TIME: DUCKDB_TYPE = 14; +pub const DUCKDB_TYPE_DUCKDB_TYPE_INTERVAL: DUCKDB_TYPE = 15; +pub const DUCKDB_TYPE_DUCKDB_TYPE_HUGEINT: DUCKDB_TYPE = 16; +pub const DUCKDB_TYPE_DUCKDB_TYPE_UHUGEINT: DUCKDB_TYPE = 32; +pub const DUCKDB_TYPE_DUCKDB_TYPE_VARCHAR: DUCKDB_TYPE = 17; +pub const DUCKDB_TYPE_DUCKDB_TYPE_BLOB: DUCKDB_TYPE = 18; +pub const DUCKDB_TYPE_DUCKDB_TYPE_DECIMAL: DUCKDB_TYPE = 19; +pub const DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_S: DUCKDB_TYPE = 20; +pub const DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_MS: DUCKDB_TYPE = 21; +pub const DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_NS: DUCKDB_TYPE = 22; +pub const DUCKDB_TYPE_DUCKDB_TYPE_ENUM: DUCKDB_TYPE = 23; +pub const DUCKDB_TYPE_DUCKDB_TYPE_LIST: DUCKDB_TYPE = 24; +pub const DUCKDB_TYPE_DUCKDB_TYPE_STRUCT: DUCKDB_TYPE = 25; +pub const DUCKDB_TYPE_DUCKDB_TYPE_MAP: DUCKDB_TYPE = 26; +pub const DUCKDB_TYPE_DUCKDB_TYPE_ARRAY: DUCKDB_TYPE = 33; +pub const DUCKDB_TYPE_DUCKDB_TYPE_UUID: DUCKDB_TYPE = 27; +pub const DUCKDB_TYPE_DUCKDB_TYPE_UNION: DUCKDB_TYPE = 28; +pub const DUCKDB_TYPE_DUCKDB_TYPE_BIT: DUCKDB_TYPE = 29; +pub const DUCKDB_TYPE_DUCKDB_TYPE_TIME_TZ: DUCKDB_TYPE = 30; +pub const DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_TZ: DUCKDB_TYPE = 31; +pub const DUCKDB_TYPE_DUCKDB_TYPE_ANY: DUCKDB_TYPE = 34; +pub const DUCKDB_TYPE_DUCKDB_TYPE_BIGNUM: DUCKDB_TYPE = 35; +pub const DUCKDB_TYPE_DUCKDB_TYPE_SQLNULL: DUCKDB_TYPE = 36; +pub const DUCKDB_TYPE_DUCKDB_TYPE_STRING_LITERAL: DUCKDB_TYPE = 37; +pub const DUCKDB_TYPE_DUCKDB_TYPE_INTEGER_LITERAL: DUCKDB_TYPE = 38; +pub const DUCKDB_TYPE_DUCKDB_TYPE_TIME_NS: DUCKDB_TYPE = 39; +pub const DUCKDB_TYPE_DUCKDB_TYPE_GEOMETRY: DUCKDB_TYPE = 40; +pub const DUCKDB_TYPE_DUCKDB_TYPE_VARIANT: DUCKDB_TYPE = 41; +#[doc = "! An enum over DuckDB's internal types."] +pub type DUCKDB_TYPE = ::std::os::raw::c_uint; +#[doc = "! An enum over DuckDB's internal types."] +pub use self::DUCKDB_TYPE as duckdb_type; +pub const duckdb_state_DuckDBSuccess: duckdb_state = 0; +pub const duckdb_state_DuckDBError: duckdb_state = 1; +#[doc = "! An enum over the returned state of different functions."] +pub type duckdb_state = ::std::os::raw::c_uint; +pub const duckdb_pending_state_DUCKDB_PENDING_RESULT_READY: duckdb_pending_state = 0; +pub const duckdb_pending_state_DUCKDB_PENDING_RESULT_NOT_READY: duckdb_pending_state = 1; +pub const duckdb_pending_state_DUCKDB_PENDING_ERROR: duckdb_pending_state = 2; +pub const duckdb_pending_state_DUCKDB_PENDING_NO_TASKS_AVAILABLE: duckdb_pending_state = 3; +#[doc = "! An enum over the pending state of a pending query result."] +pub type duckdb_pending_state = ::std::os::raw::c_uint; +pub const duckdb_result_type_DUCKDB_RESULT_TYPE_INVALID: duckdb_result_type = 0; +pub const duckdb_result_type_DUCKDB_RESULT_TYPE_CHANGED_ROWS: duckdb_result_type = 1; +pub const duckdb_result_type_DUCKDB_RESULT_TYPE_NOTHING: duckdb_result_type = 2; +pub const duckdb_result_type_DUCKDB_RESULT_TYPE_QUERY_RESULT: duckdb_result_type = 3; +#[doc = "! An enum over DuckDB's different result types."] +pub type duckdb_result_type = ::std::os::raw::c_uint; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_INVALID: duckdb_statement_type = 0; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_SELECT: duckdb_statement_type = 1; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_INSERT: duckdb_statement_type = 2; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_UPDATE: duckdb_statement_type = 3; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_EXPLAIN: duckdb_statement_type = 4; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_DELETE: duckdb_statement_type = 5; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_PREPARE: duckdb_statement_type = 6; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_CREATE: duckdb_statement_type = 7; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_EXECUTE: duckdb_statement_type = 8; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_ALTER: duckdb_statement_type = 9; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_TRANSACTION: duckdb_statement_type = 10; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_COPY: duckdb_statement_type = 11; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_ANALYZE: duckdb_statement_type = 12; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_VARIABLE_SET: duckdb_statement_type = 13; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_CREATE_FUNC: duckdb_statement_type = 14; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_DROP: duckdb_statement_type = 15; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_EXPORT: duckdb_statement_type = 16; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_PRAGMA: duckdb_statement_type = 17; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_VACUUM: duckdb_statement_type = 18; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_CALL: duckdb_statement_type = 19; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_SET: duckdb_statement_type = 20; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_LOAD: duckdb_statement_type = 21; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_RELATION: duckdb_statement_type = 22; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_EXTENSION: duckdb_statement_type = 23; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_LOGICAL_PLAN: duckdb_statement_type = 24; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_ATTACH: duckdb_statement_type = 25; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_DETACH: duckdb_statement_type = 26; +pub const duckdb_statement_type_DUCKDB_STATEMENT_TYPE_MULTI: duckdb_statement_type = 27; +#[doc = "! An enum over DuckDB's different statement types."] +pub type duckdb_statement_type = ::std::os::raw::c_uint; +pub const duckdb_error_type_DUCKDB_ERROR_INVALID: duckdb_error_type = 0; +pub const duckdb_error_type_DUCKDB_ERROR_OUT_OF_RANGE: duckdb_error_type = 1; +pub const duckdb_error_type_DUCKDB_ERROR_CONVERSION: duckdb_error_type = 2; +pub const duckdb_error_type_DUCKDB_ERROR_UNKNOWN_TYPE: duckdb_error_type = 3; +pub const duckdb_error_type_DUCKDB_ERROR_DECIMAL: duckdb_error_type = 4; +pub const duckdb_error_type_DUCKDB_ERROR_MISMATCH_TYPE: duckdb_error_type = 5; +pub const duckdb_error_type_DUCKDB_ERROR_DIVIDE_BY_ZERO: duckdb_error_type = 6; +pub const duckdb_error_type_DUCKDB_ERROR_OBJECT_SIZE: duckdb_error_type = 7; +pub const duckdb_error_type_DUCKDB_ERROR_INVALID_TYPE: duckdb_error_type = 8; +pub const duckdb_error_type_DUCKDB_ERROR_SERIALIZATION: duckdb_error_type = 9; +pub const duckdb_error_type_DUCKDB_ERROR_TRANSACTION: duckdb_error_type = 10; +pub const duckdb_error_type_DUCKDB_ERROR_NOT_IMPLEMENTED: duckdb_error_type = 11; +pub const duckdb_error_type_DUCKDB_ERROR_EXPRESSION: duckdb_error_type = 12; +pub const duckdb_error_type_DUCKDB_ERROR_CATALOG: duckdb_error_type = 13; +pub const duckdb_error_type_DUCKDB_ERROR_PARSER: duckdb_error_type = 14; +pub const duckdb_error_type_DUCKDB_ERROR_PLANNER: duckdb_error_type = 15; +pub const duckdb_error_type_DUCKDB_ERROR_SCHEDULER: duckdb_error_type = 16; +pub const duckdb_error_type_DUCKDB_ERROR_EXECUTOR: duckdb_error_type = 17; +pub const duckdb_error_type_DUCKDB_ERROR_CONSTRAINT: duckdb_error_type = 18; +pub const duckdb_error_type_DUCKDB_ERROR_INDEX: duckdb_error_type = 19; +pub const duckdb_error_type_DUCKDB_ERROR_STAT: duckdb_error_type = 20; +pub const duckdb_error_type_DUCKDB_ERROR_CONNECTION: duckdb_error_type = 21; +pub const duckdb_error_type_DUCKDB_ERROR_SYNTAX: duckdb_error_type = 22; +pub const duckdb_error_type_DUCKDB_ERROR_SETTINGS: duckdb_error_type = 23; +pub const duckdb_error_type_DUCKDB_ERROR_BINDER: duckdb_error_type = 24; +pub const duckdb_error_type_DUCKDB_ERROR_NETWORK: duckdb_error_type = 25; +pub const duckdb_error_type_DUCKDB_ERROR_OPTIMIZER: duckdb_error_type = 26; +pub const duckdb_error_type_DUCKDB_ERROR_NULL_POINTER: duckdb_error_type = 27; +pub const duckdb_error_type_DUCKDB_ERROR_IO: duckdb_error_type = 28; +pub const duckdb_error_type_DUCKDB_ERROR_INTERRUPT: duckdb_error_type = 29; +pub const duckdb_error_type_DUCKDB_ERROR_FATAL: duckdb_error_type = 30; +pub const duckdb_error_type_DUCKDB_ERROR_INTERNAL: duckdb_error_type = 31; +pub const duckdb_error_type_DUCKDB_ERROR_INVALID_INPUT: duckdb_error_type = 32; +pub const duckdb_error_type_DUCKDB_ERROR_OUT_OF_MEMORY: duckdb_error_type = 33; +pub const duckdb_error_type_DUCKDB_ERROR_PERMISSION: duckdb_error_type = 34; +pub const duckdb_error_type_DUCKDB_ERROR_PARAMETER_NOT_RESOLVED: duckdb_error_type = 35; +pub const duckdb_error_type_DUCKDB_ERROR_PARAMETER_NOT_ALLOWED: duckdb_error_type = 36; +pub const duckdb_error_type_DUCKDB_ERROR_DEPENDENCY: duckdb_error_type = 37; +pub const duckdb_error_type_DUCKDB_ERROR_HTTP: duckdb_error_type = 38; +pub const duckdb_error_type_DUCKDB_ERROR_MISSING_EXTENSION: duckdb_error_type = 39; +pub const duckdb_error_type_DUCKDB_ERROR_AUTOLOAD: duckdb_error_type = 40; +pub const duckdb_error_type_DUCKDB_ERROR_SEQUENCE: duckdb_error_type = 41; +pub const duckdb_error_type_DUCKDB_INVALID_CONFIGURATION: duckdb_error_type = 42; +#[doc = "! An enum over DuckDB's different error types."] +pub type duckdb_error_type = ::std::os::raw::c_uint; +pub const duckdb_cast_mode_DUCKDB_CAST_NORMAL: duckdb_cast_mode = 0; +pub const duckdb_cast_mode_DUCKDB_CAST_TRY: duckdb_cast_mode = 1; +#[doc = "! An enum over DuckDB's different cast modes."] +pub type duckdb_cast_mode = ::std::os::raw::c_uint; +pub const duckdb_file_flag_DUCKDB_FILE_FLAG_INVALID: duckdb_file_flag = 0; +pub const duckdb_file_flag_DUCKDB_FILE_FLAG_READ: duckdb_file_flag = 1; +pub const duckdb_file_flag_DUCKDB_FILE_FLAG_WRITE: duckdb_file_flag = 2; +pub const duckdb_file_flag_DUCKDB_FILE_FLAG_CREATE: duckdb_file_flag = 3; +pub const duckdb_file_flag_DUCKDB_FILE_FLAG_CREATE_NEW: duckdb_file_flag = 4; +pub const duckdb_file_flag_DUCKDB_FILE_FLAG_APPEND: duckdb_file_flag = 5; +pub type duckdb_file_flag = ::std::os::raw::c_uint; +pub const duckdb_config_option_scope_DUCKDB_CONFIG_OPTION_SCOPE_INVALID: duckdb_config_option_scope = 0; +pub const duckdb_config_option_scope_DUCKDB_CONFIG_OPTION_SCOPE_LOCAL: duckdb_config_option_scope = 1; +pub const duckdb_config_option_scope_DUCKDB_CONFIG_OPTION_SCOPE_SESSION: duckdb_config_option_scope = 2; +pub const duckdb_config_option_scope_DUCKDB_CONFIG_OPTION_SCOPE_GLOBAL: duckdb_config_option_scope = 3; +#[doc = "! An enum over DuckDB's configuration option scopes.\n! This enum can be used to specify the default scope when creating a custom configuration option,\n! but it is also be used to determine the scope in which a configuration option is set when it is\n! changed or retrieved."] +pub type duckdb_config_option_scope = ::std::os::raw::c_uint; +pub const duckdb_catalog_entry_type_DUCKDB_CATALOG_ENTRY_TYPE_INVALID: duckdb_catalog_entry_type = 0; +pub const duckdb_catalog_entry_type_DUCKDB_CATALOG_ENTRY_TYPE_TABLE: duckdb_catalog_entry_type = 1; +pub const duckdb_catalog_entry_type_DUCKDB_CATALOG_ENTRY_TYPE_SCHEMA: duckdb_catalog_entry_type = 2; +pub const duckdb_catalog_entry_type_DUCKDB_CATALOG_ENTRY_TYPE_VIEW: duckdb_catalog_entry_type = 3; +pub const duckdb_catalog_entry_type_DUCKDB_CATALOG_ENTRY_TYPE_INDEX: duckdb_catalog_entry_type = 4; +pub const duckdb_catalog_entry_type_DUCKDB_CATALOG_ENTRY_TYPE_PREPARED_STATEMENT: duckdb_catalog_entry_type = 5; +pub const duckdb_catalog_entry_type_DUCKDB_CATALOG_ENTRY_TYPE_SEQUENCE: duckdb_catalog_entry_type = 6; +pub const duckdb_catalog_entry_type_DUCKDB_CATALOG_ENTRY_TYPE_COLLATION: duckdb_catalog_entry_type = 7; +pub const duckdb_catalog_entry_type_DUCKDB_CATALOG_ENTRY_TYPE_TYPE: duckdb_catalog_entry_type = 8; +pub const duckdb_catalog_entry_type_DUCKDB_CATALOG_ENTRY_TYPE_DATABASE: duckdb_catalog_entry_type = 9; +#[doc = "! An enum over DuckDB's catalog entry types."] +pub type duckdb_catalog_entry_type = ::std::os::raw::c_uint; +#[doc = "! DuckDB's index type."] +pub type idx_t = u64; +#[doc = "! Type definition for the data pointers of selection vectors."] +pub type sel_t = u32; +#[doc = "! The callback to destroy data, e.g.,\n! bind data (if any), init data (if any), extra data for replacement scans (if any), etc."] +pub type duckdb_delete_callback_t = ::std::option::Option; +#[doc = "! The callback to copy data, e.g., bind data (if any)."] +pub type duckdb_copy_callback_t = + ::std::option::Option *mut ::std::os::raw::c_void>; +#[doc = "! Used for threading, contains a task state.\n! Must be destroyed with `duckdb_destroy_task_state`."] +pub type duckdb_task_state = *mut ::std::os::raw::c_void; +#[doc = "! DATE is stored as days since 1970-01-01.\n! Use the `duckdb_from_date` and `duckdb_to_date` functions to extract individual information."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_date { + pub days: i32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_date_struct { + pub year: i32, + pub month: i8, + pub day: i8, +} +#[doc = "! TIME is stored as microseconds since 00:00:00.\n! Use the `duckdb_from_time` and `duckdb_to_time` functions to extract individual information."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_time { + pub micros: i64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_time_struct { + pub hour: i8, + pub min: i8, + pub sec: i8, + pub micros: i32, +} +#[doc = "! TIME_NS is stored as nanoseconds since 00:00:00."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_time_ns { + pub nanos: i64, +} +#[doc = "! TIME_TZ is stored as 40 bits for the int64_t microseconds, and 24 bits for the int32_t offset.\n! Use the `duckdb_from_time_tz` function to extract individual information."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_time_tz { + pub bits: u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_time_tz_struct { + pub time: duckdb_time_struct, + pub offset: i32, +} +#[doc = "! TIMESTAMP is stored as microseconds since 1970-01-01.\n! Use the `duckdb_from_timestamp` and `duckdb_to_timestamp` functions to extract individual information."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_timestamp { + pub micros: i64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_timestamp_struct { + pub date: duckdb_date_struct, + pub time: duckdb_time_struct, +} +#[doc = "! TIMESTAMP_S is stored as seconds since 1970-01-01."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_timestamp_s { + pub seconds: i64, +} +#[doc = "! TIMESTAMP_MS is stored as milliseconds since 1970-01-01."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_timestamp_ms { + pub millis: i64, +} +#[doc = "! TIMESTAMP_NS is stored as nanoseconds since 1970-01-01."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_timestamp_ns { + pub nanos: i64, +} +#[doc = "! INTERVAL is stored in months, days, and micros."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_interval { + pub months: i32, + pub days: i32, + pub micros: i64, +} +#[doc = "! HUGEINT is composed of a lower and upper component.\n! Its value is upper * 2^64 + lower.\n! For simplified usage, use `duckdb_hugeint_to_double` and `duckdb_double_to_hugeint`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_hugeint { + pub lower: u64, + pub upper: i64, +} +#[doc = "! UHUGEINT is composed of a lower and upper component.\n! Its value is upper * 2^64 + lower.\n! For simplified usage, use `duckdb_uhugeint_to_double` and `duckdb_double_to_uhugeint`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_uhugeint { + pub lower: u64, + pub upper: u64, +} +#[doc = "! DECIMAL is composed of a width and a scale.\n! Their value is stored in a HUGEINT."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_decimal { + pub width: u8, + pub scale: u8, + pub value: duckdb_hugeint, +} +#[doc = "! A type holding information about the query execution progress."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_query_progress_type { + pub percentage: f64, + pub rows_processed: u64, + pub total_rows_to_process: u64, +} +#[doc = "! The internal representation of a VARCHAR (string_t). If the VARCHAR does not\n! exceed 12 characters, then we inline it. Otherwise, we inline a four-byte prefix for faster\n! string comparisons and store a pointer to the remaining characters. This is a non-\n! owning structure, i.e., it does not have to be freed."] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct duckdb_string_t { + pub value: duckdb_string_t__bindgen_ty_1, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union duckdb_string_t__bindgen_ty_1 { + pub pointer: duckdb_string_t__bindgen_ty_1__bindgen_ty_1, + pub inlined: duckdb_string_t__bindgen_ty_1__bindgen_ty_2, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_string_t__bindgen_ty_1__bindgen_ty_1 { + pub length: u32, + pub prefix: [::std::os::raw::c_char; 4usize], + pub ptr: *mut ::std::os::raw::c_char, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_string_t__bindgen_ty_1__bindgen_ty_2 { + pub length: u32, + pub inlined: [::std::os::raw::c_char; 12usize], +} +#[doc = "! DuckDB's LISTs are composed of a 'parent' vector holding metadata of each list,\n! and a child vector holding the entries of the lists.\n! The `duckdb_list_entry` struct contains the internal representation of a LIST metadata entry.\n! A metadata entry contains the length of the list, and its offset in the child vector."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_list_entry { + pub offset: u64, + pub length: u64, +} +#[doc = "! A column consists of a pointer to its internal data. Don't operate on this type directly.\n! Instead, use functions such as `duckdb_column_data`, `duckdb_nullmask_data`,\n! `duckdb_column_type`, and `duckdb_column_name`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_column { + pub deprecated_data: *mut ::std::os::raw::c_void, + pub deprecated_nullmask: *mut bool, + pub deprecated_type: duckdb_type, + pub deprecated_name: *mut ::std::os::raw::c_char, + pub internal_data: *mut ::std::os::raw::c_void, +} +#[doc = "! 1. A standalone vector that must be destroyed, or\n! 2. A vector to a column in a data chunk that lives as long as the data chunk lives."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_vector { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! 1. A standalone vector that must be destroyed, or\n! 2. A vector to a column in a data chunk that lives as long as the data chunk lives."] +pub type duckdb_vector = *mut _duckdb_vector; +#[doc = "! A selection vector is a vector of indices, which usually refer to values in a vector.\n! Can be used to slice vectors, changing their length and the order of their entries.\n! Standalone selection vectors must be destroyed."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_selection_vector { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A selection vector is a vector of indices, which usually refer to values in a vector.\n! Can be used to slice vectors, changing their length and the order of their entries.\n! Standalone selection vectors must be destroyed."] +pub type duckdb_selection_vector = *mut _duckdb_selection_vector; +#[doc = "! Strings are composed of a `char` pointer and a size.\n! You must free `string.data` with `duckdb_free`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_string { + pub data: *mut ::std::os::raw::c_char, + pub size: idx_t, +} +#[doc = "! BLOBs are composed of a byte pointer and a size.\n! You must free `blob.data` with `duckdb_free`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_blob { + pub data: *mut ::std::os::raw::c_void, + pub size: idx_t, +} +#[doc = "! BITs are composed of a byte pointer and a size.\n! BIT byte data has 0 to 7 bits of padding.\n! The first byte contains the number of padding bits.\n! The padding bits of the second byte are set to 1, starting from the MSB.\n! You must free `data` with `duckdb_free`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_bit { + pub data: *mut u8, + pub size: idx_t, +} +#[doc = "! BIGNUMs are composed of a byte pointer, a size, and an `is_negative` bool.\n! The absolute value of the number is stored in `data` in big endian format.\n! You must free `data` with `duckdb_free`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_bignum { + pub data: *mut u8, + pub size: idx_t, + pub is_negative: bool, +} +#[doc = "! A query result consists of a pointer to its internal data.\n! Must be freed with 'duckdb_destroy_result'."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_result { + pub deprecated_column_count: idx_t, + pub deprecated_row_count: idx_t, + pub deprecated_rows_changed: idx_t, + pub deprecated_columns: *mut duckdb_column, + pub deprecated_error_message: *mut ::std::os::raw::c_char, + pub internal_data: *mut ::std::os::raw::c_void, +} +#[doc = "! A database instance cache object. Must be destroyed with `duckdb_destroy_instance_cache`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_instance_cache { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A database instance cache object. Must be destroyed with `duckdb_destroy_instance_cache`."] +pub type duckdb_instance_cache = *mut _duckdb_instance_cache; +#[doc = "! A database object. Must be closed with `duckdb_close`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_database { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A database object. Must be closed with `duckdb_close`."] +pub type duckdb_database = *mut _duckdb_database; +#[doc = "! A connection to a duckdb database. Must be closed with `duckdb_disconnect`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_connection { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A connection to a duckdb database. Must be closed with `duckdb_disconnect`."] +pub type duckdb_connection = *mut _duckdb_connection; +#[doc = "! A client context of a duckdb connection. Must be destroyed with `duckdb_destroy_context`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_client_context { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A client context of a duckdb connection. Must be destroyed with `duckdb_destroy_context`."] +pub type duckdb_client_context = *mut _duckdb_client_context; +#[doc = "! A prepared statement is a parameterized query that allows you to bind parameters to it.\n! Must be destroyed with `duckdb_destroy_prepare`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_prepared_statement { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A prepared statement is a parameterized query that allows you to bind parameters to it.\n! Must be destroyed with `duckdb_destroy_prepare`."] +pub type duckdb_prepared_statement = *mut _duckdb_prepared_statement; +#[doc = "! Extracted statements. Must be destroyed with `duckdb_destroy_extracted`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_extracted_statements { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Extracted statements. Must be destroyed with `duckdb_destroy_extracted`."] +pub type duckdb_extracted_statements = *mut _duckdb_extracted_statements; +#[doc = "! The pending result represents an intermediate structure for a query that is not yet fully executed.\n! Must be destroyed with `duckdb_destroy_pending`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_pending_result { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! The pending result represents an intermediate structure for a query that is not yet fully executed.\n! Must be destroyed with `duckdb_destroy_pending`."] +pub type duckdb_pending_result = *mut _duckdb_pending_result; +#[doc = "! The appender enables fast data loading into DuckDB.\n! Must be destroyed with `duckdb_appender_destroy`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_appender { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! The appender enables fast data loading into DuckDB.\n! Must be destroyed with `duckdb_appender_destroy`."] +pub type duckdb_appender = *mut _duckdb_appender; +#[doc = "! The table description allows querying information about the table.\n! Must be destroyed with `duckdb_table_description_destroy`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_table_description { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! The table description allows querying information about the table.\n! Must be destroyed with `duckdb_table_description_destroy`."] +pub type duckdb_table_description = *mut _duckdb_table_description; +#[doc = "! The configuration can be used to provide start-up options for a database.\n! Must be destroyed with `duckdb_destroy_config`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_config { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! The configuration can be used to provide start-up options for a database.\n! Must be destroyed with `duckdb_destroy_config`."] +pub type duckdb_config = *mut _duckdb_config; +#[doc = "! A custom configuration option instance. Used to register custom options that can be set on a duckdb_config.\n! or by the user in SQL using `SET = `."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_config_option { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A custom configuration option instance. Used to register custom options that can be set on a duckdb_config.\n! or by the user in SQL using `SET = `."] +pub type duckdb_config_option = *mut _duckdb_config_option; +#[doc = "! A logical type.\n! Must be destroyed with `duckdb_destroy_logical_type`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_logical_type { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A logical type.\n! Must be destroyed with `duckdb_destroy_logical_type`."] +pub type duckdb_logical_type = *mut _duckdb_logical_type; +#[doc = "! Holds extra information to register a custom logical type.\n! Reserved for future use."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_create_type_info { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Holds extra information to register a custom logical type.\n! Reserved for future use."] +pub type duckdb_create_type_info = *mut _duckdb_create_type_info; +#[doc = "! Contains a data chunk of a duckdb_result.\n! Must be destroyed with `duckdb_destroy_data_chunk`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_data_chunk { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Contains a data chunk of a duckdb_result.\n! Must be destroyed with `duckdb_destroy_data_chunk`."] +pub type duckdb_data_chunk = *mut _duckdb_data_chunk; +#[doc = "! A value of a logical type.\n! Must be destroyed with `duckdb_destroy_value`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_value { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A value of a logical type.\n! Must be destroyed with `duckdb_destroy_value`."] +pub type duckdb_value = *mut _duckdb_value; +#[doc = "! Holds a recursive tree containing profiling metrics.\n! The tree matches the query plan, and has a top-level node."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_profiling_info { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Holds a recursive tree containing profiling metrics.\n! The tree matches the query plan, and has a top-level node."] +pub type duckdb_profiling_info = *mut _duckdb_profiling_info; +#[doc = "! Holds error data.\n! Must be destroyed with `duckdb_destroy_error_data`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_error_data { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Holds error data.\n! Must be destroyed with `duckdb_destroy_error_data`."] +pub type duckdb_error_data = *mut _duckdb_error_data; +#[doc = "! Holds a bound expression.\n! Must be destroyed with `duckdb_destroy_expression`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_expression { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Holds a bound expression.\n! Must be destroyed with `duckdb_destroy_expression`."] +pub type duckdb_expression = *mut _duckdb_expression; +#[doc = "! Holds the state of the C API extension initialization process."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_extension_info { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Holds the state of the C API extension initialization process."] +pub type duckdb_extension_info = *mut _duckdb_extension_info; +#[doc = "! Additional function info.\n! When setting this info, it is necessary to pass a destroy-callback function."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_function_info { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Additional function info.\n! When setting this info, it is necessary to pass a destroy-callback function."] +pub type duckdb_function_info = *mut _duckdb_function_info; +#[doc = "! The bind info of a function.\n! When setting this info, it is necessary to pass a destroy-callback function."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_bind_info { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! The bind info of a function.\n! When setting this info, it is necessary to pass a destroy-callback function."] +pub type duckdb_bind_info = *mut _duckdb_bind_info; +#[doc = "! Additional function initialization info.\n! When setting this info, it is necessary to pass a destroy-callback function."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_init_info { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Additional function initialization info.\n! When setting this info, it is necessary to pass a destroy-callback function."] +pub type duckdb_init_info = *mut _duckdb_init_info; +#[doc = "! A scalar function. Must be destroyed with `duckdb_destroy_scalar_function`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_scalar_function { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A scalar function. Must be destroyed with `duckdb_destroy_scalar_function`."] +pub type duckdb_scalar_function = *mut _duckdb_scalar_function; +#[doc = "! A scalar function set. Must be destroyed with `duckdb_destroy_scalar_function_set`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_scalar_function_set { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A scalar function set. Must be destroyed with `duckdb_destroy_scalar_function_set`."] +pub type duckdb_scalar_function_set = *mut _duckdb_scalar_function_set; +#[doc = "! The bind function callback of the scalar function."] +pub type duckdb_scalar_function_bind_t = ::std::option::Option; +#[doc = "! The thread-local initialization function of the scalar function."] +pub type duckdb_scalar_function_init_t = ::std::option::Option; +#[doc = "! The function to execute the scalar function on an input chunk."] +pub type duckdb_scalar_function_t = ::std::option::Option< + unsafe extern "C" fn(info: duckdb_function_info, input: duckdb_data_chunk, output: duckdb_vector), +>; +#[doc = "! An aggregate function. Must be destroyed with `duckdb_destroy_aggregate_function`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_aggregate_function { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! An aggregate function. Must be destroyed with `duckdb_destroy_aggregate_function`."] +pub type duckdb_aggregate_function = *mut _duckdb_aggregate_function; +#[doc = "! A aggregate function set. Must be destroyed with `duckdb_destroy_aggregate_function_set`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_aggregate_function_set { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A aggregate function set. Must be destroyed with `duckdb_destroy_aggregate_function_set`."] +pub type duckdb_aggregate_function_set = *mut _duckdb_aggregate_function_set; +#[doc = "! The state of an aggregate function."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_aggregate_state { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! The state of an aggregate function."] +pub type duckdb_aggregate_state = *mut _duckdb_aggregate_state; +#[doc = "! A function to return the aggregate state's size."] +pub type duckdb_aggregate_state_size = ::std::option::Option idx_t>; +#[doc = "! A function to initialize an aggregate state."] +pub type duckdb_aggregate_init_t = + ::std::option::Option; +#[doc = "! An optional function to destroy an aggregate state."] +pub type duckdb_aggregate_destroy_t = + ::std::option::Option; +#[doc = "! A function to update a set of aggregate states with new values."] +pub type duckdb_aggregate_update_t = ::std::option::Option< + unsafe extern "C" fn(info: duckdb_function_info, input: duckdb_data_chunk, states: *mut duckdb_aggregate_state), +>; +#[doc = "! A function to combine aggregate states."] +pub type duckdb_aggregate_combine_t = ::std::option::Option< + unsafe extern "C" fn( + info: duckdb_function_info, + source: *mut duckdb_aggregate_state, + target: *mut duckdb_aggregate_state, + count: idx_t, + ), +>; +#[doc = "! A function to finalize aggregate states into a result vector."] +pub type duckdb_aggregate_finalize_t = ::std::option::Option< + unsafe extern "C" fn( + info: duckdb_function_info, + source: *mut duckdb_aggregate_state, + result: duckdb_vector, + count: idx_t, + offset: idx_t, + ), +>; +#[doc = "! A table function. Must be destroyed with `duckdb_destroy_table_function`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_table_function { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A table function. Must be destroyed with `duckdb_destroy_table_function`."] +pub type duckdb_table_function = *mut _duckdb_table_function; +#[doc = "! The bind function of the table function."] +pub type duckdb_table_function_bind_t = ::std::option::Option; +#[doc = "! The possibly thread-local initialization function of the table function."] +pub type duckdb_table_function_init_t = ::std::option::Option; +#[doc = "! The function to generate an output chunk during table function execution."] +pub type duckdb_table_function_t = + ::std::option::Option; +#[doc = "! A COPY function. Must be destroyed with `duckdb_destroy_copy_function`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_copy_function { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A COPY function. Must be destroyed with `duckdb_destroy_copy_function`."] +pub type duckdb_copy_function = *mut _duckdb_copy_function; +#[doc = "! Info for the bind function of a COPY function."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_copy_function_bind_info { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Info for the bind function of a COPY function."] +pub type duckdb_copy_function_bind_info = *mut _duckdb_copy_function_bind_info; +#[doc = "! Info for the global initialization function of a COPY function."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_copy_function_global_init_info { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Info for the global initialization function of a COPY function."] +pub type duckdb_copy_function_global_init_info = *mut _duckdb_copy_function_global_init_info; +#[doc = "! Info for the sink function of a COPY function."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_copy_function_sink_info { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Info for the sink function of a COPY function."] +pub type duckdb_copy_function_sink_info = *mut _duckdb_copy_function_sink_info; +#[doc = "! Info for the finalize function of a COPY function."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_copy_function_finalize_info { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Info for the finalize function of a COPY function."] +pub type duckdb_copy_function_finalize_info = *mut _duckdb_copy_function_finalize_info; +#[doc = "! The bind function to use when binding a COPY ... TO function."] +pub type duckdb_copy_function_bind_t = + ::std::option::Option; +#[doc = "! The initialization function to use when initializing a COPY ... TO function."] +pub type duckdb_copy_function_global_init_t = + ::std::option::Option; +#[doc = "! The function to sink an input chunk into during execution of a COPY ... TO function."] +pub type duckdb_copy_function_sink_t = + ::std::option::Option; +#[doc = "! The function to finalize the COPY ... TO function execution."] +pub type duckdb_copy_function_finalize_t = + ::std::option::Option; +#[doc = "! A cast function. Must be destroyed with `duckdb_destroy_cast_function`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_cast_function { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A cast function. Must be destroyed with `duckdb_destroy_cast_function`."] +pub type duckdb_cast_function = *mut _duckdb_cast_function; +#[doc = "! The function to cast from an input vector to an output vector."] +pub type duckdb_cast_function_t = ::std::option::Option< + unsafe extern "C" fn(info: duckdb_function_info, count: idx_t, input: duckdb_vector, output: duckdb_vector) -> bool, +>; +#[doc = "! Additional replacement scan info. When setting this info, it is necessary to pass a destroy-callback function."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_replacement_scan_info { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Additional replacement scan info. When setting this info, it is necessary to pass a destroy-callback function."] +pub type duckdb_replacement_scan_info = *mut _duckdb_replacement_scan_info; +#[doc = "! A replacement scan function."] +pub type duckdb_replacement_callback_t = ::std::option::Option< + unsafe extern "C" fn( + info: duckdb_replacement_scan_info, + table_name: *const ::std::os::raw::c_char, + data: *mut ::std::os::raw::c_void, + ), +>; +#[doc = "! Holds an arrow query result. Must be destroyed with `duckdb_destroy_arrow`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_arrow { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Holds an arrow query result. Must be destroyed with `duckdb_destroy_arrow`."] +pub type duckdb_arrow = *mut _duckdb_arrow; +#[doc = "! Holds an arrow array stream. Must be destroyed with `duckdb_destroy_arrow_stream`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_arrow_stream { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Holds an arrow array stream. Must be destroyed with `duckdb_destroy_arrow_stream`."] +pub type duckdb_arrow_stream = *mut _duckdb_arrow_stream; +#[doc = "! Holds an arrow schema. Remember to release the respective ArrowSchema object."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_arrow_schema { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Holds an arrow schema. Remember to release the respective ArrowSchema object."] +pub type duckdb_arrow_schema = *mut _duckdb_arrow_schema; +#[doc = "! Holds an arrow converted schema (i.e., duckdb::ArrowTableSchema).\n! In practice, this object holds the information necessary to do proper conversion between Arrow Types and DuckDB\n! Types. Check duckdb/function/table/arrow/arrow_duck_schema.hpp for more details! Must be destroyed with\n! `duckdb_destroy_arrow_converted_schema`"] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_arrow_converted_schema { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Holds an arrow converted schema (i.e., duckdb::ArrowTableSchema).\n! In practice, this object holds the information necessary to do proper conversion between Arrow Types and DuckDB\n! Types. Check duckdb/function/table/arrow/arrow_duck_schema.hpp for more details! Must be destroyed with\n! `duckdb_destroy_arrow_converted_schema`"] +pub type duckdb_arrow_converted_schema = *mut _duckdb_arrow_converted_schema; +#[doc = "! Holds an arrow array. Remember to release the respective ArrowSchema object."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_arrow_array { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Holds an arrow array. Remember to release the respective ArrowSchema object."] +pub type duckdb_arrow_array = *mut _duckdb_arrow_array; +#[doc = "! The arrow options used when transforming the DuckDB schema and datachunks into Arrow schema and arrays.\n! Used in `duckdb_to_arrow_schema` and `duckdb_data_chunk_to_arrow`"] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_arrow_options { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! The arrow options used when transforming the DuckDB schema and datachunks into Arrow schema and arrays.\n! Used in `duckdb_to_arrow_schema` and `duckdb_data_chunk_to_arrow`"] +pub type duckdb_arrow_options = *mut _duckdb_arrow_options; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_file_open_options { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +pub type duckdb_file_open_options = *mut _duckdb_file_open_options; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_file_system { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +pub type duckdb_file_system = *mut _duckdb_file_system; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_file_handle { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +pub type duckdb_file_handle = *mut _duckdb_file_handle; +#[doc = "! A handle to a database catalog.\n! Must be destroyed with `duckdb_destroy_catalog`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_catalog { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A handle to a database catalog.\n! Must be destroyed with `duckdb_destroy_catalog`."] +pub type duckdb_catalog = *mut _duckdb_catalog; +#[doc = "! A handle to a catalog entry (e.g., table, view, index, etc.).\n! Must be destroyed with `duckdb_destroy_catalog_entry`."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_catalog_entry { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! A handle to a catalog entry (e.g., table, view, index, etc.).\n! Must be destroyed with `duckdb_destroy_catalog_entry`."] +pub type duckdb_catalog_entry = *mut _duckdb_catalog_entry; +#[doc = "! Holds a log storage object."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _duckdb_log_storage { + pub internal_ptr: *mut ::std::os::raw::c_void, +} +#[doc = "! Holds a log storage object."] +pub type duckdb_log_storage = *mut _duckdb_log_storage; +#[doc = "! This function is missing the logging context, which will be added later."] +pub type duckdb_logger_write_log_entry_t = ::std::option::Option< + unsafe extern "C" fn( + extra_data: *mut ::std::os::raw::c_void, + timestamp: *mut duckdb_timestamp, + level: *const ::std::os::raw::c_char, + log_type: *const ::std::os::raw::c_char, + log_message: *const ::std::os::raw::c_char, + ), +>; +#[doc = "! Passed to C API extension as a parameter to the entrypoint."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct duckdb_extension_access { + #[doc = "! Indicate that an error has occurred."] + pub set_error: + ::std::option::Option, + #[doc = "! Fetch the database on which to register the extension."] + pub get_database: ::std::option::Option *mut duckdb_database>, + #[doc = "! Fetch the API struct pointer."] + pub get_api: ::std::option::Option< + unsafe extern "C" fn( + info: duckdb_extension_info, + version: *const ::std::os::raw::c_char, + ) -> *const ::std::os::raw::c_void, + >, +} +unsafe extern "C" { + #[doc = "Creates a new database instance cache.\nThe instance cache is necessary if a client/program (re)opens multiple databases to the same file within the same\nprocess. Must be destroyed with 'duckdb_destroy_instance_cache'.\n\n @return The database instance cache."] + pub fn duckdb_create_instance_cache() -> duckdb_instance_cache; +} +unsafe extern "C" { + #[doc = "Creates a new database instance in the instance cache, or retrieves an existing database instance.\nMust be closed with 'duckdb_close'.\n\n @param instance_cache The instance cache in which to create the database, or from which to take the database.\n @param path Path to the database file on disk. Both `nullptr` and `:memory:` open or retrieve an in-memory database.\n @param out_database The resulting cached database.\n @param config (Optional) configuration used to create the database.\n @param out_error If set and the function returns `DuckDBError`, this contains the error message.\nNote that the error message must be freed using `duckdb_free`.\n @return `DuckDBSuccess` on success or `DuckDBError` on failure."] + pub fn duckdb_get_or_create_from_cache( + instance_cache: duckdb_instance_cache, + path: *const ::std::os::raw::c_char, + out_database: *mut duckdb_database, + config: duckdb_config, + out_error: *mut *mut ::std::os::raw::c_char, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Destroys an existing database instance cache and de-allocates its memory.\n\n @param instance_cache The instance cache to destroy."] + pub fn duckdb_destroy_instance_cache(instance_cache: *mut duckdb_instance_cache); +} +unsafe extern "C" { + #[doc = "Creates a new database or opens an existing database file stored at the given path.\nIf no path is given a new in-memory database is created instead.\nThe database must be closed with 'duckdb_close'.\n\n @param path Path to the database file on disk. Both `nullptr` and `:memory:` open an in-memory database.\n @param out_database The result database object.\n @return `DuckDBSuccess` on success or `DuckDBError` on failure."] + pub fn duckdb_open(path: *const ::std::os::raw::c_char, out_database: *mut duckdb_database) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Extended version of duckdb_open. Creates a new database or opens an existing database file stored at the given path.\nThe database must be closed with 'duckdb_close'.\n\n @param path Path to the database file on disk. Both `nullptr` and `:memory:` open an in-memory database.\n @param out_database The result database object.\n @param config (Optional) configuration used to start up the database.\n @param out_error If set and the function returns `DuckDBError`, this contains the error message.\nNote that the error message must be freed using `duckdb_free`.\n @return `DuckDBSuccess` on success or `DuckDBError` on failure."] + pub fn duckdb_open_ext( + path: *const ::std::os::raw::c_char, + out_database: *mut duckdb_database, + config: duckdb_config, + out_error: *mut *mut ::std::os::raw::c_char, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Closes the specified database and de-allocates all memory allocated for that database.\nThis should be called after you are done with any database allocated through `duckdb_open` or `duckdb_open_ext`.\nNote that failing to call `duckdb_close` (in case of e.g. a program crash) will not cause data corruption.\nStill, it is recommended to always correctly close a database object after you are done with it.\n\n @param database The database object to shut down."] + pub fn duckdb_close(database: *mut duckdb_database); +} +unsafe extern "C" { + #[doc = "Opens a connection to a database. Connections are required to query the database, and store transactional state\nassociated with the connection.\nThe instantiated connection should be closed using 'duckdb_disconnect'.\n\n @param database The database file to connect to.\n @param out_connection The result connection object.\n @return `DuckDBSuccess` on success or `DuckDBError` on failure."] + pub fn duckdb_connect(database: duckdb_database, out_connection: *mut duckdb_connection) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Interrupt running query\n\n @param connection The connection to interrupt"] + pub fn duckdb_interrupt(connection: duckdb_connection); +} +unsafe extern "C" { + #[doc = "Get the progress of the running query.\n\n @param connection The connection running the query.\n @return The query progress type containing progress information."] + pub fn duckdb_query_progress(connection: duckdb_connection) -> duckdb_query_progress_type; +} +unsafe extern "C" { + #[doc = "Closes the specified connection and de-allocates all memory allocated for that connection.\n\n @param connection The connection to close."] + pub fn duckdb_disconnect(connection: *mut duckdb_connection); +} +unsafe extern "C" { + #[doc = "Retrieves the client context of the connection.\n\n @param connection The connection.\n @param out_context The client context of the connection. Must be destroyed with `duckdb_destroy_client_context`."] + pub fn duckdb_connection_get_client_context(connection: duckdb_connection, out_context: *mut duckdb_client_context); +} +unsafe extern "C" { + #[doc = "Retrieves the arrow options of the connection.\n\n @param connection The connection."] + pub fn duckdb_connection_get_arrow_options( + connection: duckdb_connection, + out_arrow_options: *mut duckdb_arrow_options, + ); +} +unsafe extern "C" { + #[doc = "Returns the connection id of the client context.\n\n @param context The client context.\n @return The connection id of the client context."] + pub fn duckdb_client_context_get_connection_id(context: duckdb_client_context) -> idx_t; +} +unsafe extern "C" { + #[doc = "Destroys the client context and deallocates its memory.\n\n @param context The client context to destroy."] + pub fn duckdb_destroy_client_context(context: *mut duckdb_client_context); +} +unsafe extern "C" { + #[doc = "Destroys the arrow options and deallocates its memory.\n\n @param arrow_options The arrow options to destroy."] + pub fn duckdb_destroy_arrow_options(arrow_options: *mut duckdb_arrow_options); +} +unsafe extern "C" { + #[doc = "Returns the version of the linked DuckDB, with a version postfix for dev versions\n\nUsually used for developing C extensions that must return this for a compatibility check."] + pub fn duckdb_library_version() -> *const ::std::os::raw::c_char; +} +unsafe extern "C" { + #[doc = "Get the list of (fully qualified) table names of the query.\n\n @param connection The connection for which to get the table names.\n @param query The query for which to get the table names.\n @param qualified Returns fully qualified table names (catalog.schema.table), if set to true, else only the (not\nescaped) table names.\n @return A duckdb_value of type VARCHAR[] containing the (fully qualified) table names of the query. Must be destroyed\nwith duckdb_destroy_value."] + pub fn duckdb_get_table_names( + connection: duckdb_connection, + query: *const ::std::os::raw::c_char, + qualified: bool, + ) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Initializes an empty configuration object that can be used to provide start-up options for the DuckDB instance\nthrough `duckdb_open_ext`.\nThe duckdb_config must be destroyed using 'duckdb_destroy_config'\n\nThis will always succeed unless there is a malloc failure.\n\nNote that `duckdb_destroy_config` should always be called on the resulting config, even if the function returns\n`DuckDBError`.\n\n @param out_config The result configuration object.\n @return `DuckDBSuccess` on success or `DuckDBError` on failure."] + pub fn duckdb_create_config(out_config: *mut duckdb_config) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "This returns the total amount of configuration options available for usage with `duckdb_get_config_flag`.\n\nThis should not be called in a loop as it internally loops over all the options.\n\n @return The amount of config options available."] + pub fn duckdb_config_count() -> usize; +} +unsafe extern "C" { + #[doc = "Obtains a human-readable name and description of a specific configuration option. This can be used to e.g.\ndisplay configuration options. This will succeed unless `index` is out of range (i.e. `>= duckdb_config_count`).\n\nThe result name or description MUST NOT be freed.\n\n @param index The index of the configuration option (between 0 and `duckdb_config_count`)\n @param out_name A name of the configuration flag.\n @param out_description A description of the configuration flag.\n @return `DuckDBSuccess` on success or `DuckDBError` on failure."] + pub fn duckdb_get_config_flag( + index: usize, + out_name: *mut *const ::std::os::raw::c_char, + out_description: *mut *const ::std::os::raw::c_char, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Sets the specified option for the specified configuration. The configuration option is indicated by name.\nTo obtain a list of config options, see `duckdb_get_config_flag`.\n\nIn the source code, configuration options are defined in `config.cpp`.\n\nThis can fail if either the name is invalid, or if the value provided for the option is invalid.\n\n @param config The configuration object to set the option on.\n @param name The name of the configuration flag to set.\n @param option The value to set the configuration flag to.\n @return `DuckDBSuccess` on success or `DuckDBError` on failure."] + pub fn duckdb_set_config( + config: duckdb_config, + name: *const ::std::os::raw::c_char, + option: *const ::std::os::raw::c_char, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Destroys the specified configuration object and de-allocates all memory allocated for the object.\n\n @param config The configuration object to destroy."] + pub fn duckdb_destroy_config(config: *mut duckdb_config); +} +unsafe extern "C" { + #[doc = "Creates duckdb_error_data.\nMust be destroyed with `duckdb_destroy_error_data`.\n\n @param type The error type.\n @param message The error message.\n @return The error data."] + pub fn duckdb_create_error_data( + type_: duckdb_error_type, + message: *const ::std::os::raw::c_char, + ) -> duckdb_error_data; +} +unsafe extern "C" { + #[doc = "Destroys the error data and deallocates its memory.\n\n @param error_data The error data to destroy."] + pub fn duckdb_destroy_error_data(error_data: *mut duckdb_error_data); +} +unsafe extern "C" { + #[doc = "Returns the duckdb_error_type of the error data.\n\n @param error_data The error data.\n @return The error type."] + pub fn duckdb_error_data_error_type(error_data: duckdb_error_data) -> duckdb_error_type; +} +unsafe extern "C" { + #[doc = "Returns the error message of the error data. Must not be freed.\n\n @param error_data The error data.\n @return The error message."] + pub fn duckdb_error_data_message(error_data: duckdb_error_data) -> *const ::std::os::raw::c_char; +} +unsafe extern "C" { + #[doc = "Returns whether the error data contains an error or not.\n\n @param error_data The error data.\n @return True, if the error data contains an exception, else false."] + pub fn duckdb_error_data_has_error(error_data: duckdb_error_data) -> bool; +} +unsafe extern "C" { + #[doc = "Executes a SQL query within a connection and stores the full (materialized) result in the out_result pointer.\nIf the query fails to execute, DuckDBError is returned and the error message can be retrieved by calling\n`duckdb_result_error`.\n\nNote that after running `duckdb_query`, `duckdb_destroy_result` must be called on the result object even if the\nquery fails, otherwise the error stored within the result will not be freed correctly.\n\n @param connection The connection to perform the query in.\n @param query The SQL query to run.\n @param out_result The query result.\n @return `DuckDBSuccess` on success or `DuckDBError` on failure."] + pub fn duckdb_query( + connection: duckdb_connection, + query: *const ::std::os::raw::c_char, + out_result: *mut duckdb_result, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Closes the result and de-allocates all memory allocated for that result.\n\n @param result The result to destroy."] + pub fn duckdb_destroy_result(result: *mut duckdb_result); +} +unsafe extern "C" { + #[doc = "Returns the column name of the specified column. The result should not need to be freed; the column names will\nautomatically be destroyed when the result is destroyed.\n\nReturns `NULL` if the column is out of range.\n\n @param result The result object to fetch the column name from.\n @param col The column index.\n @return The column name of the specified column."] + pub fn duckdb_column_name(result: *mut duckdb_result, col: idx_t) -> *const ::std::os::raw::c_char; +} +unsafe extern "C" { + #[doc = "Returns the column type of the specified column.\n\nReturns `DUCKDB_TYPE_INVALID` if the column is out of range.\n\n @param result The result object to fetch the column type from.\n @param col The column index.\n @return The column type of the specified column."] + pub fn duckdb_column_type(result: *mut duckdb_result, col: idx_t) -> duckdb_type; +} +unsafe extern "C" { + #[doc = "Returns the statement type of the statement that was executed\n\n @param result The result object to fetch the statement type from.\n @return duckdb_statement_type value or DUCKDB_STATEMENT_TYPE_INVALID"] + pub fn duckdb_result_statement_type(result: duckdb_result) -> duckdb_statement_type; +} +unsafe extern "C" { + #[doc = "Returns the logical column type of the specified column.\n\nThe return type of this call should be destroyed with `duckdb_destroy_logical_type`.\n\nReturns `NULL` if the column is out of range.\n\n @param result The result object to fetch the column type from.\n @param col The column index.\n @return The logical column type of the specified column."] + pub fn duckdb_column_logical_type(result: *mut duckdb_result, col: idx_t) -> duckdb_logical_type; +} +unsafe extern "C" { + #[doc = "Returns the arrow options associated with the given result. These options are definitions of how the arrow arrays/schema\nshould be produced.\n @param result The result object to fetch arrow options from.\n @return The arrow options associated with the given result. This must be destroyed with\n`duckdb_destroy_arrow_options`."] + pub fn duckdb_result_get_arrow_options(result: *mut duckdb_result) -> duckdb_arrow_options; +} +unsafe extern "C" { + #[doc = "Returns the number of columns present in a the result object.\n\n @param result The result object.\n @return The number of columns present in the result object."] + pub fn duckdb_column_count(result: *mut duckdb_result) -> idx_t; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\nReturns the number of rows present in the result object.\n\n @param result The result object.\n @return The number of rows present in the result object."] + pub fn duckdb_row_count(result: *mut duckdb_result) -> idx_t; +} +unsafe extern "C" { + #[doc = "Returns the number of rows changed by the query stored in the result. This is relevant only for INSERT/UPDATE/DELETE\nqueries. For other queries the rows_changed will be 0.\n\n @param result The result object.\n @return The number of rows changed."] + pub fn duckdb_rows_changed(result: *mut duckdb_result) -> idx_t; +} +unsafe extern "C" { + #[doc = "DEPRECATED**: Prefer using `duckdb_result_get_chunk` instead.\n\nReturns the data of a specific column of a result in columnar format.\n\nThe function returns a dense array which contains the result data. The exact type stored in the array depends on the\ncorresponding duckdb_type (as provided by `duckdb_column_type`). For the exact type by which the data should be\naccessed, see the comments in [the types section](types) or the `DUCKDB_TYPE` enum.\n\nFor example, for a column of type `DUCKDB_TYPE_INTEGER`, rows can be accessed in the following manner:\n```c\nint32_t *data = (int32_t *) duckdb_column_data(&result, 0);\nprintf(\"Data for row %d: %d\\n\", row, data[row]);\n```\n\n @param result The result object to fetch the column data from.\n @param col The column index.\n @return The column data of the specified column."] + pub fn duckdb_column_data(result: *mut duckdb_result, col: idx_t) -> *mut ::std::os::raw::c_void; +} +unsafe extern "C" { + #[doc = "DEPRECATED**: Prefer using `duckdb_result_get_chunk` instead.\n\nReturns the nullmask of a specific column of a result in columnar format. The nullmask indicates for every row\nwhether or not the corresponding row is `NULL`. If a row is `NULL`, the values present in the array provided\nby `duckdb_column_data` are undefined.\n\n```c\nint32_t *data = (int32_t *) duckdb_column_data(&result, 0);\nbool *nullmask = duckdb_nullmask_data(&result, 0);\nif (nullmask[row]) {\nprintf(\"Data for row %d: NULL\\n\", row);\n} else {\nprintf(\"Data for row %d: %d\\n\", row, data[row]);\n}\n```\n\n @param result The result object to fetch the nullmask from.\n @param col The column index.\n @return The nullmask of the specified column."] + pub fn duckdb_nullmask_data(result: *mut duckdb_result, col: idx_t) -> *mut bool; +} +unsafe extern "C" { + #[doc = "Returns the error message contained within the result. The error is only set if `duckdb_query` returns `DuckDBError`.\n\nThe result of this function must not be freed. It will be cleaned up when `duckdb_destroy_result` is called.\n\n @param result The result object to fetch the error from.\n @return The error of the result."] + pub fn duckdb_result_error(result: *mut duckdb_result) -> *const ::std::os::raw::c_char; +} +unsafe extern "C" { + #[doc = "Returns the result error type contained within the result. The error is only set if `duckdb_query` returns\n`DuckDBError`.\n\n @param result The result object to fetch the error from.\n @return The error type of the result."] + pub fn duckdb_result_error_type(result: *mut duckdb_result) -> duckdb_error_type; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\nFetches a data chunk from the duckdb_result. This function should be called repeatedly until the result is exhausted.\n\nThe result must be destroyed with `duckdb_destroy_data_chunk`.\n\nThis function supersedes all `duckdb_value` functions, as well as the `duckdb_column_data` and `duckdb_nullmask_data`\nfunctions. It results in significantly better performance, and should be preferred in newer code-bases.\n\nIf this function is used, none of the other result functions can be used and vice versa (i.e. this function cannot be\nmixed with the legacy result functions).\n\nUse `duckdb_result_chunk_count` to figure out how many chunks there are in the result.\n\n @param result The result object to fetch the data chunk from.\n @param chunk_index The chunk index to fetch from.\n @return The resulting data chunk. Returns `NULL` if the chunk index is out of bounds."] + pub fn duckdb_result_get_chunk(result: duckdb_result, chunk_index: idx_t) -> duckdb_data_chunk; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\nChecks if the type of the internal result is StreamQueryResult.\n\n @param result The result object to check.\n @return Whether or not the result object is of the type StreamQueryResult"] + pub fn duckdb_result_is_streaming(result: duckdb_result) -> bool; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\nReturns the number of data chunks present in the result.\n\n @param result The result object\n @return Number of data chunks present in the result."] + pub fn duckdb_result_chunk_count(result: duckdb_result) -> idx_t; +} +unsafe extern "C" { + #[doc = "Returns the return_type of the given result, or DUCKDB_RETURN_TYPE_INVALID on error\n\n @param result The result object\n @return The return_type"] + pub fn duckdb_result_return_type(result: duckdb_result) -> duckdb_result_type; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The boolean value at the specified location, or false if the value cannot be converted."] + pub fn duckdb_value_boolean(result: *mut duckdb_result, col: idx_t, row: idx_t) -> bool; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The int8_t value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_int8(result: *mut duckdb_result, col: idx_t, row: idx_t) -> i8; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The int16_t value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_int16(result: *mut duckdb_result, col: idx_t, row: idx_t) -> i16; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The int32_t value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_int32(result: *mut duckdb_result, col: idx_t, row: idx_t) -> i32; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The int64_t value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_int64(result: *mut duckdb_result, col: idx_t, row: idx_t) -> i64; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The duckdb_hugeint value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_hugeint(result: *mut duckdb_result, col: idx_t, row: idx_t) -> duckdb_hugeint; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The duckdb_uhugeint value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_uhugeint(result: *mut duckdb_result, col: idx_t, row: idx_t) -> duckdb_uhugeint; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The duckdb_decimal value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_decimal(result: *mut duckdb_result, col: idx_t, row: idx_t) -> duckdb_decimal; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The uint8_t value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_uint8(result: *mut duckdb_result, col: idx_t, row: idx_t) -> u8; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The uint16_t value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_uint16(result: *mut duckdb_result, col: idx_t, row: idx_t) -> u16; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The uint32_t value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_uint32(result: *mut duckdb_result, col: idx_t, row: idx_t) -> u32; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The uint64_t value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_uint64(result: *mut duckdb_result, col: idx_t, row: idx_t) -> u64; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The float value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_float(result: *mut duckdb_result, col: idx_t, row: idx_t) -> f32; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The double value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_double(result: *mut duckdb_result, col: idx_t, row: idx_t) -> f64; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The duckdb_date value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_date(result: *mut duckdb_result, col: idx_t, row: idx_t) -> duckdb_date; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The duckdb_time value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_time(result: *mut duckdb_result, col: idx_t, row: idx_t) -> duckdb_time; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The duckdb_timestamp value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_timestamp(result: *mut duckdb_result, col: idx_t, row: idx_t) -> duckdb_timestamp; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The duckdb_interval value at the specified location, or 0 if the value cannot be converted."] + pub fn duckdb_value_interval(result: *mut duckdb_result, col: idx_t, row: idx_t) -> duckdb_interval; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The text value at the specified location as a null-terminated string, or nullptr if the value cannot be\nconverted. The result must be freed with `duckdb_free`."] + pub fn duckdb_value_varchar(result: *mut duckdb_result, col: idx_t, row: idx_t) -> *mut ::std::os::raw::c_char; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The string value at the specified location. Attempts to cast the result value to string."] + pub fn duckdb_value_string(result: *mut duckdb_result, col: idx_t, row: idx_t) -> duckdb_string; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The char* value at the specified location. ONLY works on VARCHAR columns and does not auto-cast.\nIf the column is NOT a VARCHAR column this function will return NULL.\n\nThe result must NOT be freed."] + pub fn duckdb_value_varchar_internal( + result: *mut duckdb_result, + col: idx_t, + row: idx_t, + ) -> *mut ::std::os::raw::c_char; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The char* value at the specified location. ONLY works on VARCHAR columns and does not auto-cast.\nIf the column is NOT a VARCHAR column this function will return NULL.\n\nThe result must NOT be freed."] + pub fn duckdb_value_string_internal(result: *mut duckdb_result, col: idx_t, row: idx_t) -> duckdb_string; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return The duckdb_blob value at the specified location. Returns a blob with blob.data set to nullptr if the\nvalue cannot be converted. The resulting field \"blob.data\" must be freed with `duckdb_free.`"] + pub fn duckdb_value_blob(result: *mut duckdb_result, col: idx_t, row: idx_t) -> duckdb_blob; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n @return Returns true if the value at the specified index is NULL, and false otherwise."] + pub fn duckdb_value_is_null(result: *mut duckdb_result, col: idx_t, row: idx_t) -> bool; +} +unsafe extern "C" { + #[doc = "Allocate `size` bytes of memory using the duckdb internal malloc function. Any memory allocated in this manner\nshould be freed using `duckdb_free`.\n\n @param size The number of bytes to allocate.\n @return A pointer to the allocated memory region."] + pub fn duckdb_malloc(size: usize) -> *mut ::std::os::raw::c_void; +} +unsafe extern "C" { + #[doc = "Free a value returned from `duckdb_malloc`, `duckdb_value_varchar`, `duckdb_value_blob`, or\n`duckdb_value_string`.\n\n @param ptr The memory region to de-allocate."] + pub fn duckdb_free(ptr: *mut ::std::os::raw::c_void); +} +unsafe extern "C" { + #[doc = "The internal vector size used by DuckDB.\nThis is the amount of tuples that will fit into a data chunk created by `duckdb_create_data_chunk`.\n\n @return The vector size."] + pub fn duckdb_vector_size() -> idx_t; +} +unsafe extern "C" { + #[doc = "Whether or not the duckdb_string_t value is inlined.\nThis means that the data of the string does not have a separate allocation."] + pub fn duckdb_string_is_inlined(string: duckdb_string_t) -> bool; +} +unsafe extern "C" { + #[doc = "Get the string length of a string_t\n\n @param string The string to get the length of.\n @return The length."] + pub fn duckdb_string_t_length(string: duckdb_string_t) -> u32; +} +unsafe extern "C" { + #[doc = "Get a pointer to the string data of a string_t\n\n @param string The string to get the pointer to.\n @return The pointer."] + pub fn duckdb_string_t_data(string: *mut duckdb_string_t) -> *const ::std::os::raw::c_char; +} +unsafe extern "C" { + #[doc = "Checks if a string is valid UTF-8.\n\n @param str The string to check\n @param len The length of the string (in bytes)\n @return nullptr if the string is valid UTF-8. Otherwise, a duckdb_error_data containing error information. Must be\ndestroyed with `duckdb_destroy_error_data`."] + pub fn duckdb_valid_utf8_check(str_: *const ::std::os::raw::c_char, len: idx_t) -> duckdb_error_data; +} +unsafe extern "C" { + #[doc = "Decompose a `duckdb_date` object into year, month and date (stored as `duckdb_date_struct`).\n\n @param date The date object, as obtained from a `DUCKDB_TYPE_DATE` column.\n @return The `duckdb_date_struct` with the decomposed elements."] + pub fn duckdb_from_date(date: duckdb_date) -> duckdb_date_struct; +} +unsafe extern "C" { + #[doc = "Re-compose a `duckdb_date` from year, month and date (`duckdb_date_struct`).\n\n @param date The year, month and date stored in a `duckdb_date_struct`.\n @return The `duckdb_date` element."] + pub fn duckdb_to_date(date: duckdb_date_struct) -> duckdb_date; +} +unsafe extern "C" { + #[doc = "Test a `duckdb_date` to see if it is a finite value.\n\n @param date The date object, as obtained from a `DUCKDB_TYPE_DATE` column.\n @return True if the date is finite, false if it is ±infinity."] + pub fn duckdb_is_finite_date(date: duckdb_date) -> bool; +} +unsafe extern "C" { + #[doc = "Decompose a `duckdb_time` object into hour, minute, second and microsecond (stored as `duckdb_time_struct`).\n\n @param time The time object, as obtained from a `DUCKDB_TYPE_TIME` column.\n @return The `duckdb_time_struct` with the decomposed elements."] + pub fn duckdb_from_time(time: duckdb_time) -> duckdb_time_struct; +} +unsafe extern "C" { + #[doc = "Create a `duckdb_time_tz` object from micros and a timezone offset.\n\n @param micros The microsecond component of the time.\n @param offset The timezone offset component of the time.\n @return The `duckdb_time_tz` element."] + pub fn duckdb_create_time_tz(micros: i64, offset: i32) -> duckdb_time_tz; +} +unsafe extern "C" { + #[doc = "Decompose a TIME_TZ objects into micros and a timezone offset.\n\nUse `duckdb_from_time` to further decompose the micros into hour, minute, second and microsecond.\n\n @param micros The time object, as obtained from a `DUCKDB_TYPE_TIME_TZ` column."] + pub fn duckdb_from_time_tz(micros: duckdb_time_tz) -> duckdb_time_tz_struct; +} +unsafe extern "C" { + #[doc = "Re-compose a `duckdb_time` from hour, minute, second and microsecond (`duckdb_time_struct`).\n\n @param time The hour, minute, second and microsecond in a `duckdb_time_struct`.\n @return The `duckdb_time` element."] + pub fn duckdb_to_time(time: duckdb_time_struct) -> duckdb_time; +} +unsafe extern "C" { + #[doc = "Decompose a `duckdb_timestamp` object into a `duckdb_timestamp_struct`.\n\n @param ts The ts object, as obtained from a `DUCKDB_TYPE_TIMESTAMP` column.\n @return The `duckdb_timestamp_struct` with the decomposed elements."] + pub fn duckdb_from_timestamp(ts: duckdb_timestamp) -> duckdb_timestamp_struct; +} +unsafe extern "C" { + #[doc = "Re-compose a `duckdb_timestamp` from a duckdb_timestamp_struct.\n\n @param ts The de-composed elements in a `duckdb_timestamp_struct`.\n @return The `duckdb_timestamp` element."] + pub fn duckdb_to_timestamp(ts: duckdb_timestamp_struct) -> duckdb_timestamp; +} +unsafe extern "C" { + #[doc = "Test a `duckdb_timestamp` to see if it is a finite value.\n\n @param ts The duckdb_timestamp object, as obtained from a `DUCKDB_TYPE_TIMESTAMP` column.\n @return True if the timestamp is finite, false if it is ±infinity."] + pub fn duckdb_is_finite_timestamp(ts: duckdb_timestamp) -> bool; +} +unsafe extern "C" { + #[doc = "Test a `duckdb_timestamp_s` to see if it is a finite value.\n\n @param ts The duckdb_timestamp_s object, as obtained from a `DUCKDB_TYPE_TIMESTAMP_S` column.\n @return True if the timestamp is finite, false if it is ±infinity."] + pub fn duckdb_is_finite_timestamp_s(ts: duckdb_timestamp_s) -> bool; +} +unsafe extern "C" { + #[doc = "Test a `duckdb_timestamp_ms` to see if it is a finite value.\n\n @param ts The duckdb_timestamp_ms object, as obtained from a `DUCKDB_TYPE_TIMESTAMP_MS` column.\n @return True if the timestamp is finite, false if it is ±infinity."] + pub fn duckdb_is_finite_timestamp_ms(ts: duckdb_timestamp_ms) -> bool; +} +unsafe extern "C" { + #[doc = "Test a `duckdb_timestamp_ns` to see if it is a finite value.\n\n @param ts The duckdb_timestamp_ns object, as obtained from a `DUCKDB_TYPE_TIMESTAMP_NS` column.\n @return True if the timestamp is finite, false if it is ±infinity."] + pub fn duckdb_is_finite_timestamp_ns(ts: duckdb_timestamp_ns) -> bool; +} +unsafe extern "C" { + #[doc = "Converts a duckdb_hugeint object (as obtained from a `DUCKDB_TYPE_HUGEINT` column) into a double.\n\n @param val The hugeint value.\n @return The converted `double` element."] + pub fn duckdb_hugeint_to_double(val: duckdb_hugeint) -> f64; +} +unsafe extern "C" { + #[doc = "Converts a double value to a duckdb_hugeint object.\n\nIf the conversion fails because the double value is too big the result will be 0.\n\n @param val The double value.\n @return The converted `duckdb_hugeint` element."] + pub fn duckdb_double_to_hugeint(val: f64) -> duckdb_hugeint; +} +unsafe extern "C" { + #[doc = "Converts a duckdb_uhugeint object (as obtained from a `DUCKDB_TYPE_UHUGEINT` column) into a double.\n\n @param val The uhugeint value.\n @return The converted `double` element."] + pub fn duckdb_uhugeint_to_double(val: duckdb_uhugeint) -> f64; +} +unsafe extern "C" { + #[doc = "Converts a double value to a duckdb_uhugeint object.\n\nIf the conversion fails because the double value is too big the result will be 0.\n\n @param val The double value.\n @return The converted `duckdb_uhugeint` element."] + pub fn duckdb_double_to_uhugeint(val: f64) -> duckdb_uhugeint; +} +unsafe extern "C" { + #[doc = "Converts a double value to a duckdb_decimal object.\n\nIf the conversion fails because the double value is too big, or the width/scale are invalid the result will be 0.\n\n @param val The double value.\n @return The converted `duckdb_decimal` element."] + pub fn duckdb_double_to_decimal(val: f64, width: u8, scale: u8) -> duckdb_decimal; +} +unsafe extern "C" { + #[doc = "Converts a duckdb_decimal object (as obtained from a `DUCKDB_TYPE_DECIMAL` column) into a double.\n\n @param val The decimal value.\n @return The converted `double` element."] + pub fn duckdb_decimal_to_double(val: duckdb_decimal) -> f64; +} +unsafe extern "C" { + #[doc = "Create a prepared statement object from a query.\n\nNote that after calling `duckdb_prepare`, the prepared statement should always be destroyed using\n`duckdb_destroy_prepare`, even if the prepare fails.\n\nIf the prepare fails, `duckdb_prepare_error` can be called to obtain the reason why the prepare failed.\n\n @param connection The connection object\n @param query The SQL query to prepare\n @param out_prepared_statement The resulting prepared statement object\n @return `DuckDBSuccess` on success or `DuckDBError` on failure."] + pub fn duckdb_prepare( + connection: duckdb_connection, + query: *const ::std::os::raw::c_char, + out_prepared_statement: *mut duckdb_prepared_statement, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Closes the prepared statement and de-allocates all memory allocated for the statement.\n\n @param prepared_statement The prepared statement to destroy."] + pub fn duckdb_destroy_prepare(prepared_statement: *mut duckdb_prepared_statement); +} +unsafe extern "C" { + #[doc = "Returns the error message associated with the given prepared statement.\nIf the prepared statement has no error message, this returns `nullptr` instead.\n\nThe error message should not be freed. It will be de-allocated when `duckdb_destroy_prepare` is called.\n\n @param prepared_statement The prepared statement to obtain the error from.\n @return The error message, or `nullptr` if there is none."] + pub fn duckdb_prepare_error(prepared_statement: duckdb_prepared_statement) -> *const ::std::os::raw::c_char; +} +unsafe extern "C" { + #[doc = "Returns the number of parameters that can be provided to the given prepared statement.\n\nReturns 0 if the query was not successfully prepared.\n\n @param prepared_statement The prepared statement to obtain the number of parameters for."] + pub fn duckdb_nparams(prepared_statement: duckdb_prepared_statement) -> idx_t; +} +unsafe extern "C" { + #[doc = "Returns the name used to identify the parameter\nThe returned string should be freed using `duckdb_free`.\n\nReturns NULL if the index is out of range for the provided prepared statement.\n\n @param prepared_statement The prepared statement for which to get the parameter name from."] + pub fn duckdb_parameter_name( + prepared_statement: duckdb_prepared_statement, + index: idx_t, + ) -> *const ::std::os::raw::c_char; +} +unsafe extern "C" { + #[doc = "Returns the parameter type for the parameter at the given index.\n\nReturns `DUCKDB_TYPE_INVALID` if the parameter index is out of range or the statement was not successfully prepared.\n\n @param prepared_statement The prepared statement.\n @param param_idx The parameter index.\n @return The parameter type"] + pub fn duckdb_param_type(prepared_statement: duckdb_prepared_statement, param_idx: idx_t) -> duckdb_type; +} +unsafe extern "C" { + #[doc = "Returns the logical type for the parameter at the given index.\n\nReturns `nullptr` if the parameter index is out of range or the statement was not successfully prepared.\n\nThe return type of this call should be destroyed with `duckdb_destroy_logical_type`.\n\n @param prepared_statement The prepared statement.\n @param param_idx The parameter index.\n @return The logical type of the parameter"] + pub fn duckdb_param_logical_type( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + ) -> duckdb_logical_type; +} +unsafe extern "C" { + #[doc = "Clear the params bind to the prepared statement."] + pub fn duckdb_clear_bindings(prepared_statement: duckdb_prepared_statement) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Returns the statement type of the statement to be executed\n\n @param statement The prepared statement.\n @return duckdb_statement_type value or DUCKDB_STATEMENT_TYPE_INVALID"] + pub fn duckdb_prepared_statement_type(statement: duckdb_prepared_statement) -> duckdb_statement_type; +} +unsafe extern "C" { + #[doc = "Returns the number of columns present in a the result of the prepared statement. If any of the column types are invalid,\nthe result will be 1.\n\n @param prepared_statement The prepared statement.\n @return The number of columns present in the result of the prepared statement."] + pub fn duckdb_prepared_statement_column_count(prepared_statement: duckdb_prepared_statement) -> idx_t; +} +unsafe extern "C" { + #[doc = "Returns the name of the specified column of the result of the prepared_statement.\nThe returned string should be freed using `duckdb_free`.\n\nReturns `nullptr` if the column is out of range.\n\n @param prepared_statement The prepared statement.\n @param col_idx The column index.\n @return The column name of the specified column."] + pub fn duckdb_prepared_statement_column_name( + prepared_statement: duckdb_prepared_statement, + col_idx: idx_t, + ) -> *const ::std::os::raw::c_char; +} +unsafe extern "C" { + #[doc = "Returns the column type of the specified column of the result of the prepared_statement.\n\nReturns `DUCKDB_TYPE_INVALID` if the column is out of range.\nThe return type of this call should be destroyed with `duckdb_destroy_logical_type`.\n\n @param prepared_statement The prepared statement to fetch the column type from.\n @param col_idx The column index.\n @return The logical type of the specified column."] + pub fn duckdb_prepared_statement_column_logical_type( + prepared_statement: duckdb_prepared_statement, + col_idx: idx_t, + ) -> duckdb_logical_type; +} +unsafe extern "C" { + #[doc = "Returns the column type of the specified column of the result of the prepared_statement.\n\nReturns `DUCKDB_TYPE_INVALID` if the column is out of range.\n\n @param prepared_statement The prepared statement to fetch the column type from.\n @param col_idx The column index.\n @return The type of the specified column."] + pub fn duckdb_prepared_statement_column_type( + prepared_statement: duckdb_prepared_statement, + col_idx: idx_t, + ) -> duckdb_type; +} +unsafe extern "C" { + #[doc = "Binds a value to the prepared statement at the specified index.\n\nSupersedes all type-specific bind functions (e.g., `duckdb_bind_varchar`, `duckdb_bind_int64`, etc.)."] + pub fn duckdb_bind_value( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: duckdb_value, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Retrieve the index of the parameter for the prepared statement, identified by name"] + pub fn duckdb_bind_parameter_index( + prepared_statement: duckdb_prepared_statement, + param_idx_out: *mut idx_t, + name: *const ::std::os::raw::c_char, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a bool value to the prepared statement at the specified index."] + pub fn duckdb_bind_boolean( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: bool, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds an int8_t value to the prepared statement at the specified index."] + pub fn duckdb_bind_int8(prepared_statement: duckdb_prepared_statement, param_idx: idx_t, val: i8) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds an int16_t value to the prepared statement at the specified index."] + pub fn duckdb_bind_int16(prepared_statement: duckdb_prepared_statement, param_idx: idx_t, val: i16) + -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds an int32_t value to the prepared statement at the specified index."] + pub fn duckdb_bind_int32(prepared_statement: duckdb_prepared_statement, param_idx: idx_t, val: i32) + -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds an int64_t value to the prepared statement at the specified index."] + pub fn duckdb_bind_int64(prepared_statement: duckdb_prepared_statement, param_idx: idx_t, val: i64) + -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a duckdb_hugeint value to the prepared statement at the specified index."] + pub fn duckdb_bind_hugeint( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: duckdb_hugeint, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a duckdb_uhugeint value to the prepared statement at the specified index."] + pub fn duckdb_bind_uhugeint( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: duckdb_uhugeint, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a duckdb_decimal value to the prepared statement at the specified index."] + pub fn duckdb_bind_decimal( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: duckdb_decimal, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a uint8_t value to the prepared statement at the specified index."] + pub fn duckdb_bind_uint8(prepared_statement: duckdb_prepared_statement, param_idx: idx_t, val: u8) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a uint16_t value to the prepared statement at the specified index."] + pub fn duckdb_bind_uint16( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: u16, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a uint32_t value to the prepared statement at the specified index."] + pub fn duckdb_bind_uint32( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: u32, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a uint64_t value to the prepared statement at the specified index."] + pub fn duckdb_bind_uint64( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: u64, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a float value to the prepared statement at the specified index."] + pub fn duckdb_bind_float(prepared_statement: duckdb_prepared_statement, param_idx: idx_t, val: f32) + -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a double value to the prepared statement at the specified index."] + pub fn duckdb_bind_double( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: f64, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a duckdb_date value to the prepared statement at the specified index."] + pub fn duckdb_bind_date( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: duckdb_date, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a duckdb_time value to the prepared statement at the specified index."] + pub fn duckdb_bind_time( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: duckdb_time, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a duckdb_timestamp value to the prepared statement at the specified index."] + pub fn duckdb_bind_timestamp( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: duckdb_timestamp, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a duckdb_timestamp value to the prepared statement at the specified index."] + pub fn duckdb_bind_timestamp_tz( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: duckdb_timestamp, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a duckdb_interval value to the prepared statement at the specified index."] + pub fn duckdb_bind_interval( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: duckdb_interval, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a null-terminated varchar value to the prepared statement at the specified index.\n\nSuperseded by `duckdb_bind_value`."] + pub fn duckdb_bind_varchar( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: *const ::std::os::raw::c_char, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a varchar value to the prepared statement at the specified index.\n\nSuperseded by `duckdb_bind_value`."] + pub fn duckdb_bind_varchar_length( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + val: *const ::std::os::raw::c_char, + length: idx_t, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a blob value to the prepared statement at the specified index."] + pub fn duckdb_bind_blob( + prepared_statement: duckdb_prepared_statement, + param_idx: idx_t, + data: *const ::std::os::raw::c_void, + length: idx_t, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Binds a NULL value to the prepared statement at the specified index."] + pub fn duckdb_bind_null(prepared_statement: duckdb_prepared_statement, param_idx: idx_t) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Executes the prepared statement with the given bound parameters, and returns a materialized query result.\n\nThis method can be called multiple times for each prepared statement, and the parameters can be modified\nbetween calls to this function.\n\nNote that the result must be freed with `duckdb_destroy_result`.\n\n @param prepared_statement The prepared statement to execute.\n @param out_result The query result.\n @return `DuckDBSuccess` on success or `DuckDBError` on failure."] + pub fn duckdb_execute_prepared( + prepared_statement: duckdb_prepared_statement, + out_result: *mut duckdb_result, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\nExecutes the prepared statement with the given bound parameters, and returns an optionally-streaming query result.\nTo determine if the resulting query was in fact streamed, use `duckdb_result_is_streaming`\n\nThis method can be called multiple times for each prepared statement, and the parameters can be modified\nbetween calls to this function.\n\nNote that the result must be freed with `duckdb_destroy_result`.\n\n @param prepared_statement The prepared statement to execute.\n @param out_result The query result.\n @return `DuckDBSuccess` on success or `DuckDBError` on failure."] + pub fn duckdb_execute_prepared_streaming( + prepared_statement: duckdb_prepared_statement, + out_result: *mut duckdb_result, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Extract all statements from a query.\nNote that after calling `duckdb_extract_statements`, the extracted statements should always be destroyed using\n`duckdb_destroy_extracted`, even if no statements were extracted.\n\nIf the extract fails, `duckdb_extract_statements_error` can be called to obtain the reason why the extract failed.\n\n @param connection The connection object\n @param query The SQL query to extract\n @param out_extracted_statements The resulting extracted statements object\n @return The number of extracted statements or 0 on failure."] + pub fn duckdb_extract_statements( + connection: duckdb_connection, + query: *const ::std::os::raw::c_char, + out_extracted_statements: *mut duckdb_extracted_statements, + ) -> idx_t; +} +unsafe extern "C" { + #[doc = "Prepare an extracted statement.\nNote that after calling `duckdb_prepare_extracted_statement`, the prepared statement should always be destroyed using\n`duckdb_destroy_prepare`, even if the prepare fails.\n\nIf the prepare fails, `duckdb_prepare_error` can be called to obtain the reason why the prepare failed.\n\n @param connection The connection object\n @param extracted_statements The extracted statements object\n @param index The index of the extracted statement to prepare\n @param out_prepared_statement The resulting prepared statement object\n @return `DuckDBSuccess` on success or `DuckDBError` on failure."] + pub fn duckdb_prepare_extracted_statement( + connection: duckdb_connection, + extracted_statements: duckdb_extracted_statements, + index: idx_t, + out_prepared_statement: *mut duckdb_prepared_statement, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Returns the error message contained within the extracted statements.\nThe result of this function must not be freed. It will be cleaned up when `duckdb_destroy_extracted` is called.\n\n @param extracted_statements The extracted statements to fetch the error from.\n @return The error of the extracted statements."] + pub fn duckdb_extract_statements_error( + extracted_statements: duckdb_extracted_statements, + ) -> *const ::std::os::raw::c_char; +} +unsafe extern "C" { + #[doc = "De-allocates all memory allocated for the extracted statements.\n @param extracted_statements The extracted statements to destroy."] + pub fn duckdb_destroy_extracted(extracted_statements: *mut duckdb_extracted_statements); +} +unsafe extern "C" { + #[doc = "Executes the prepared statement with the given bound parameters, and returns a pending result.\nThe pending result represents an intermediate structure for a query that is not yet fully executed.\nThe pending result can be used to incrementally execute a query, returning control to the client between tasks.\n\nNote that after calling `duckdb_pending_prepared`, the pending result should always be destroyed using\n`duckdb_destroy_pending`, even if this function returns DuckDBError.\n\n @param prepared_statement The prepared statement to execute.\n @param out_result The pending query result.\n @return `DuckDBSuccess` on success or `DuckDBError` on failure."] + pub fn duckdb_pending_prepared( + prepared_statement: duckdb_prepared_statement, + out_result: *mut duckdb_pending_result, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\nExecutes the prepared statement with the given bound parameters, and returns a pending result.\nThis pending result will create a streaming duckdb_result when executed.\nThe pending result represents an intermediate structure for a query that is not yet fully executed.\n\nNote that after calling `duckdb_pending_prepared_streaming`, the pending result should always be destroyed using\n`duckdb_destroy_pending`, even if this function returns DuckDBError.\n\n @param prepared_statement The prepared statement to execute.\n @param out_result The pending query result.\n @return `DuckDBSuccess` on success or `DuckDBError` on failure."] + pub fn duckdb_pending_prepared_streaming( + prepared_statement: duckdb_prepared_statement, + out_result: *mut duckdb_pending_result, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Closes the pending result and de-allocates all memory allocated for the result.\n\n @param pending_result The pending result to destroy."] + pub fn duckdb_destroy_pending(pending_result: *mut duckdb_pending_result); +} +unsafe extern "C" { + #[doc = "Returns the error message contained within the pending result.\n\nThe result of this function must not be freed. It will be cleaned up when `duckdb_destroy_pending` is called.\n\n @param pending_result The pending result to fetch the error from.\n @return The error of the pending result."] + pub fn duckdb_pending_error(pending_result: duckdb_pending_result) -> *const ::std::os::raw::c_char; +} +unsafe extern "C" { + #[doc = "Executes a single task within the query, returning whether or not the query is ready.\n\nIf this returns DUCKDB_PENDING_RESULT_READY, the duckdb_execute_pending function can be called to obtain the result.\nIf this returns DUCKDB_PENDING_RESULT_NOT_READY, the duckdb_pending_execute_task function should be called again.\nIf this returns DUCKDB_PENDING_ERROR, an error occurred during execution.\n\nThe error message can be obtained by calling duckdb_pending_error on the pending_result.\n\n @param pending_result The pending result to execute a task within.\n @return The state of the pending result after the execution."] + pub fn duckdb_pending_execute_task(pending_result: duckdb_pending_result) -> duckdb_pending_state; +} +unsafe extern "C" { + #[doc = "If this returns DUCKDB_PENDING_RESULT_READY, the duckdb_execute_pending function can be called to obtain the result.\nIf this returns DUCKDB_PENDING_RESULT_NOT_READY, the duckdb_pending_execute_check_state function should be called again.\nIf this returns DUCKDB_PENDING_ERROR, an error occurred during execution.\n\nThe error message can be obtained by calling duckdb_pending_error on the pending_result.\n\n @param pending_result The pending result.\n @return The state of the pending result."] + pub fn duckdb_pending_execute_check_state(pending_result: duckdb_pending_result) -> duckdb_pending_state; +} +unsafe extern "C" { + #[doc = "Fully execute a pending query result, returning the final query result.\n\nIf duckdb_pending_execute_task has been called until DUCKDB_PENDING_RESULT_READY was returned, this will return fast.\nOtherwise, all remaining tasks must be executed first.\n\nNote that the result must be freed with `duckdb_destroy_result`.\n\n @param pending_result The pending result to execute.\n @param out_result The result object.\n @return `DuckDBSuccess` on success or `DuckDBError` on failure."] + pub fn duckdb_execute_pending( + pending_result: duckdb_pending_result, + out_result: *mut duckdb_result, + ) -> duckdb_state; +} +unsafe extern "C" { + #[doc = "Returns whether a duckdb_pending_state is finished executing. For example if `pending_state` is\nDUCKDB_PENDING_RESULT_READY, this function will return true.\n\n @param pending_state The pending state on which to decide whether to finish execution.\n @return Boolean indicating pending execution should be considered finished."] + pub fn duckdb_pending_execution_is_finished(pending_state: duckdb_pending_state) -> bool; +} +unsafe extern "C" { + #[doc = "Destroys the value and de-allocates all memory allocated for that type.\n\n @param value The value to destroy."] + pub fn duckdb_destroy_value(value: *mut duckdb_value); +} +unsafe extern "C" { + #[doc = "Creates a value from a null-terminated string. Returns nullptr if the string is not valid UTF-8 or other invalid input.\n\nSuperseded by `duckdb_create_varchar_length`.\n\n @param text The null-terminated string\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_varchar(text: *const ::std::os::raw::c_char) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from a string. Returns nullptr if the string is not valid UTF-8 or other invalid input.\n\n @param text The text\n @param length The length of the text\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_varchar_length(text: *const ::std::os::raw::c_char, length: idx_t) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from a boolean\n\n @param input The boolean value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_bool(input: bool) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from an int8_t (a tinyint)\n\n @param input The tinyint value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_int8(input: i8) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from a uint8_t (a utinyint)\n\n @param input The utinyint value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_uint8(input: u8) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from an int16_t (a smallint)\n\n @param input The smallint value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_int16(input: i16) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from a uint16_t (a usmallint)\n\n @param input The usmallint value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_uint16(input: u16) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from an int32_t (an integer)\n\n @param input The integer value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_int32(input: i32) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from a uint32_t (a uinteger)\n\n @param input The uinteger value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_uint32(input: u32) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from a uint64_t (a ubigint)\n\n @param input The ubigint value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_uint64(input: u64) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from an int64\n\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_int64(val: i64) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from a hugeint\n\n @param input The hugeint value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_hugeint(input: duckdb_hugeint) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from a uhugeint\n\n @param input The uhugeint value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_uhugeint(input: duckdb_uhugeint) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a BIGNUM value from a duckdb_bignum\n\n @param input The duckdb_bignum value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_bignum(input: duckdb_bignum) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a DECIMAL value from a duckdb_decimal\n\nThe width must be between 1 and 38, and the scale must not exceed the width.\n\n @param input The duckdb_decimal value\n @return The value, or `nullptr` if the width or scale are out of range. This must be destroyed with\n`duckdb_destroy_value`."] + pub fn duckdb_create_decimal(input: duckdb_decimal) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from a float\n\n @param input The float value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_float(input: f32) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from a double\n\n @param input The double value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_double(input: f64) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from a date\n\n @param input The date value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_date(input: duckdb_date) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from a time\n\n @param input The time value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_time(input: duckdb_time) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from a time_ns\n\n @param input The time value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_time_ns(input: duckdb_time_ns) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from a time_tz.\nNot to be confused with `duckdb_create_time_tz`, which creates a duckdb_time_tz_t.\n\n @param value The time_tz value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_time_tz_value(value: duckdb_time_tz) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a TIMESTAMP value from a duckdb_timestamp\n\n @param input The duckdb_timestamp value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_timestamp(input: duckdb_timestamp) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a TIMESTAMP_TZ value from a duckdb_timestamp\n\n @param input The duckdb_timestamp value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_timestamp_tz(input: duckdb_timestamp) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a TIMESTAMP_S value from a duckdb_timestamp_s\n\n @param input The duckdb_timestamp_s value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_timestamp_s(input: duckdb_timestamp_s) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a TIMESTAMP_MS value from a duckdb_timestamp_ms\n\n @param input The duckdb_timestamp_ms value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_timestamp_ms(input: duckdb_timestamp_ms) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a TIMESTAMP_NS value from a duckdb_timestamp_ns\n\n @param input The duckdb_timestamp_ns value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_timestamp_ns(input: duckdb_timestamp_ns) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from an interval\n\n @param input The interval value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_interval(input: duckdb_interval) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a value from a blob\n\n @param data The blob data\n @param length The length of the blob data\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_blob(data: *const u8, length: idx_t) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a BIT value from a duckdb_bit\n\n @param input The duckdb_bit value\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_bit(input: duckdb_bit) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Creates a UUID value from a uhugeint\n\n @param input The duckdb_uhugeint containing the UUID\n @return The value. This must be destroyed with `duckdb_destroy_value`."] + pub fn duckdb_create_uuid(input: duckdb_uhugeint) -> duckdb_value; +} +unsafe extern "C" { + #[doc = "Returns the boolean value of the given value.\n\n @param val A duckdb_value containing a boolean\n @return A boolean, or false if the value cannot be converted"] + pub fn duckdb_get_bool(val: duckdb_value) -> bool; +} +unsafe extern "C" { + #[doc = "Returns the int8_t value of the given value.\n\n @param val A duckdb_value containing a tinyint\n @return A int8_t, or MinValue if the value cannot be converted"] + pub fn duckdb_get_int8(val: duckdb_value) -> i8; +} +unsafe extern "C" { + #[doc = "Returns the uint8_t value of the given value.\n\n @param val A duckdb_value containing a utinyint\n @return A uint8_t, or MinValue if the value cannot be converted"] + pub fn duckdb_get_uint8(val: duckdb_value) -> u8; +} +unsafe extern "C" { + #[doc = "Returns the int16_t value of the given value.\n\n @param val A duckdb_value containing a smallint\n @return A int16_t, or MinValue if the value cannot be converted"] + pub fn duckdb_get_int16(val: duckdb_value) -> i16; +} +unsafe extern "C" { + #[doc = "Returns the uint16_t value of the given value.\n\n @param val A duckdb_value containing a usmallint\n @return A uint16_t, or MinValue if the value cannot be converted"] + pub fn duckdb_get_uint16(val: duckdb_value) -> u16; +} +unsafe extern "C" { + #[doc = "Returns the int32_t value of the given value.\n\n @param val A duckdb_value containing an integer\n @return A int32_t, or MinValue if the value cannot be converted"] + pub fn duckdb_get_int32(val: duckdb_value) -> i32; +} +unsafe extern "C" { + #[doc = "Returns the uint32_t value of the given value.\n\n @param val A duckdb_value containing a uinteger\n @return A uint32_t, or MinValue if the value cannot be converted"] + pub fn duckdb_get_uint32(val: duckdb_value) -> u32; +} +unsafe extern "C" { + #[doc = "Returns the int64_t value of the given value.\n\n @param val A duckdb_value containing a bigint\n @return A int64_t, or MinValue if the value cannot be converted"] + pub fn duckdb_get_int64(val: duckdb_value) -> i64; +} +unsafe extern "C" { + #[doc = "Returns the uint64_t value of the given value.\n\n @param val A duckdb_value containing a ubigint\n @return A uint64_t, or MinValue if the value cannot be converted"] + pub fn duckdb_get_uint64(val: duckdb_value) -> u64; +} +unsafe extern "C" { + #[doc = "Returns the hugeint value of the given value.\n\n @param val A duckdb_value containing a hugeint\n @return A duckdb_hugeint, or MinValue if the value cannot be converted"] + pub fn duckdb_get_hugeint(val: duckdb_value) -> duckdb_hugeint; +} +unsafe extern "C" { + #[doc = "Returns the uhugeint value of the given value.\n\n @param val A duckdb_value containing a uhugeint\n @return A duckdb_uhugeint, or MinValue if the value cannot be converted"] + pub fn duckdb_get_uhugeint(val: duckdb_value) -> duckdb_uhugeint; +} +unsafe extern "C" { + #[doc = "Returns the duckdb_bignum value of the given value.\nThe `data` field must be destroyed with `duckdb_free`.\n\n @param val A duckdb_value containing a BIGNUM\n @return A duckdb_bignum. The `data` field must be destroyed with `duckdb_free`."] + pub fn duckdb_get_bignum(val: duckdb_value) -> duckdb_bignum; +} +unsafe extern "C" { + #[doc = "Returns the duckdb_decimal value of the given value.\n\n @param val A duckdb_value containing a DECIMAL\n @return A duckdb_decimal, or MinValue if the value cannot be converted"] + pub fn duckdb_get_decimal(val: duckdb_value) -> duckdb_decimal; +} +unsafe extern "C" { + #[doc = "Returns the float value of the given value.\n\n @param val A duckdb_value containing a float\n @return A float, or NAN if the value cannot be converted"] + pub fn duckdb_get_float(val: duckdb_value) -> f32; +} +unsafe extern "C" { + #[doc = "Returns the double value of the given value.\n\n @param val A duckdb_value containing a double\n @return A double, or NAN if the value cannot be converted"] + pub fn duckdb_get_double(val: duckdb_value) -> f64; +} +unsafe extern "C" { + #[doc = "Returns the date value of the given value.\n\n @param val A duckdb_value containing a date\n @return A duckdb_date, or MinValue if the value cannot be converted"] + pub fn duckdb_get_date(val: duckdb_value) -> duckdb_date; +} +unsafe extern "C" { + #[doc = "Returns the time value of the given value.\n\n @param val A duckdb_value containing a time\n @return A duckdb_time, or MinValue