Reduce research rebuilds and isolate DuckDB download dependencies

.github/scripts/build-research-recipes.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # Compile owners concurrently. Keep each owner's recipes and target writes serial.
 set -euo pipefail
-[[ $# == 2 || $# == 5 ]] || exit 2
+[[ $# == 2 || $# == 6 ]] || exit 2
 product=${1:?product required}
 phase=${2:?recipe measurement phase required}
 [[ $phase == after-cache-lookup || $phase == warm-local || $phase == dependency-warm-serial || $phase == dependency-warm-parallel ]] || exit 2
@@ -58,7 +58,7 @@ if [[ $# == 2 ]]; then
   trap 'cancel_scheduler 143' TERM
   # Background execution lets Bash handle a signal sent only to its own PID.
   # The Ruby supervisor receives that signal and reaps its owned process groups.
-  ruby -rjson - "$root/.github/scripts/build-research-recipes.sh" "$product" "$phase" "$inputs" "$cpus" "$memory_mib" <<'RUBY' &
+  ruby -rjson -rtmpdir -rfileutils - "$root/.github/scripts/build-research-recipes.sh" "$product" "$phase" "$inputs" "$cpus" "$memory_mib" <<'RUBY' &
 script, product, phase, inputs, cpus, memory = ARGV
 cpus, memory = Integer(cpus), Integer(memory)
 requested = ENV.fetch('MONDAY_RESEARCH_BUILD_CONCURRENCY', '2')
@@ -71,6 +71,8 @@ owners = JSON.parse(File.read(inputs)).fetch('recipes').map { |recipe| recipe.fe
 concurrency = [Integer(requested), owners.length, budget, [memory / 4096, 1].max].min
 jobs = budget / concurrency
 pending, active = owners.dup, {}
+lease_root = Dir.mktmpdir('research-recipe-jobs.', ENV.fetch('RUNNER_TEMP'))
+leases = {}
 failed_groups = []
 status = 0
 interrupted = nil
@@ -85,13 +87,25 @@ begin
   until pending.empty? && active.empty?
     while !interrupted && active.length < concurrency && !pending.empty?
       owner = pending.shift
-      pid = Process.spawn('bash', script, product, phase, owner, jobs.to_s, concurrency.to_s, pgroup: true)
+      lease = File.join(lease_root, owner.delete_suffix('/Cargo.toml').tr('/', '-') + '.jobs')
+      File.write(lease, jobs.to_s)
+      pid = Process.spawn('bash', script, product, phase, owner, jobs.to_s, concurrency.to_s, lease, pgroup: true)
       active[pid] = owner
+      leases[pid] = lease
     end
     if interrupted
       status = 128 + Signal.list.fetch(interrupted)
       break
     end
+    # Increase only the final owner's next recipe. No pending or sibling owner
+    # can claim these CPUs. An in-flight Cargo/cc jobserver keeps its old budget.
+    if pending.empty? && active.length == 1
+      lease = leases.fetch(active.keys.first)
+      unless File.read(lease) == budget.to_s
+        File.write(lease + '.next', budget.to_s)
+        File.rename(lease + '.next', lease)
+      end
+    end
     completed = Process.wait2(-1, Process::WNOHANG)
     unless completed
       IO.select([wake_read])
@@ -114,6 +128,7 @@ ensure
   sleep 0.2 unless groups.empty?
   groups.each { |pid| Process.kill('KILL', -pid) rescue Errno::ESRCH }
   groups.each { |pid| Process.wait(pid) rescue Errno::ECHILD }
+  FileUtils.remove_entry(lease_root)
   wake_read.close
   wake_write.close
 end
@@ -147,31 +162,42 @@ RUBY
   [[ $(sha256sum "$inputs" | awk '{print $1}') == "$inputs_sha" ]]
   exit "$status"
 fi
-manifest=$3 cargo_jobs=$4 concurrency=$5
+manifest=$3 cargo_jobs=$4 concurrency=$5 lease=$6
 [[ $cargo_jobs =~ ^[1-9][0-9]*$ && $concurrency =~ ^[12]$ ]]
 (( cargo_jobs * concurrency <= cpus ))
 jq -e --arg manifest "$manifest" 'any(.[]; .manifest==$manifest)' <<<"$recipes" >/dev/null
 owner=${manifest%/Cargo.toml}
 timings="$work/$phase-${owner//\//--}.timings.jsonl"
 while IFS= read -r recipe; do
+  # The supervisor grants a larger budget only after all other owners exit.
+  # Read once per recipe; never change Cargo's budget during compilation.
+  next_jobs=$(cat "$lease")
+  [[ $next_jobs =~ ^[1-9][0-9]*$ ]]
+  (( next_jobs >= cargo_jobs && next_jobs <= cpus ))
+  cargo_jobs=$next_jobs
   package=$(jq -r .package <<<"$recipe")
-  args=(build --manifest-path "$manifest" --target x86_64-unknown-linux-gnu --release --locked --jobs "$cargo_jobs" -p "$package")
-  features=$(jq -r .features <<<"$recipe")
-  [[ -z $features ]] || args+=(--features "$features")
-  while IFS= read -r binary; do args+=(--bin "$binary"); done < <(jq -r '.binaries[]' <<<"$recipe")
   start=$(date +%s)
   recipe_target=$(bash "$root/.github/scripts/research-cache-layout.sh" target-dir "$(jq -r .manifest <<<"$recipe")")
-  CARGO_TARGET_DIR="$recipe_target" cargo "${args[@]}" --message-format json-render-diagnostics >"$work/$phase-$package.jsonl"
+  # Cargo emits dirty reasons to stderr; keep stdout as compiler JSON only.
+  # Logging belongs to orchestration, not the dependency compilation identity.
+  fingerprint_log="$work/$phase-$package.fingerprint.log"
+  status=0
+  { CARGO_LOG=cargo::core::compiler::fingerprint=info CARGO_TERM_COLOR=never \
+      CARGO_TARGET_DIR="$recipe_target" bash "$root/.github/scripts/compile-research-recipe.sh" "$recipe" "$cargo_jobs" \
+      >"$work/$phase-$package.jsonl"; } 2>&1 | tee "$fingerprint_log" >&2 || status=$?
+  (( status == 0 )) || exit "$status"
   jq -es 'any(.[]; .reason=="compiler-artifact") and any(.[]; .reason=="build-finished" and .success==true)' \
     "$work/$phase-$package.jsonl" >/dev/null
   seconds=$(( $(date +%s) - start ))
   jq -s -c --arg cache_match "$cache_match" --arg phase "$phase" --arg source "$source_sha" --arg inputs "$inputs_sha" \
     --argjson requested_concurrency "${MONDAY_RESEARCH_BUILD_CONCURRENCY:-2}" \
     --argjson cargo_jobs "$cargo_jobs" --argjson concurrency "$concurrency" \
+    --rawfile diagnostics "$fingerprint_log" \
     --argjson runner "$runner_context" --arg recipe_target "$recipe_target" --argjson recipe "$recipe" --argjson seconds "$seconds" \
     '{phase:$phase,runner:$runner,recipe_target:$recipe_target,cache_exact_match:$cache_match,source_sha:$source,compilation_inputs_sha256:$inputs,recipe:$recipe,seconds:$seconds,
       scheduler:{requested_owner_concurrency:$requested_concurrency,owner_concurrency:$concurrency,cargo_jobs:$cargo_jobs},
       compiler_artifacts:([.[]|select(.reason=="compiler-artifact")]|length),
+      native_fingerprint_diagnostics:[$diagnostics|split("\n")[]|select(contains("fingerprint") and contains("libduckdb"))],
       native_artifacts:[.[]|select(.reason=="compiler-artifact" and (.package_id|contains("libduckdb-sys")))|{package_id,target:.target.name,fresh}],
       fresh:([.[]|select(.reason=="compiler-artifact" and .fresh==true)]|length),
       rebuilt:([.[]|select(.reason=="compiler-artifact" and .fresh==false)]|length)}' \

.github/scripts/capture-research-build-inputs.sh

@@ -4,6 +4,7 @@
 set -euo pipefail
 output=${1:?output JSON required}
 root=$(cd "$(dirname "$0")/../.." && pwd)
+bash "$root/.github/scripts/prepare-research-native-source.sh"
 work=$(mktemp -d)
 trap 'rm -rf "$work"' EXIT
 test "$(uname -s)" = Linux
@@ -29,9 +30,19 @@ for variable in CC CXX AR LD CFLAGS CXXFLAGS LDFLAGS RUSTC_BOOTSTRAP CARGO_BUILD
   LIBRARY_PATH LD_LIBRARY_PATH CPATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH \
   PKG_CONFIG PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR \
   OPENSSL_DIR OPENSSL_LIB_DIR OPENSSL_INCLUDE_DIR OPENSSL_STATIC \
-  BINDGEN_EXTRA_CLANG_ARGS PROTOC PROTOC_INCLUDE CMAKE_TOOLCHAIN_FILE; do
+  BINDGEN_EXTRA_CLANG_ARGS PROTOC PROTOC_INCLUDE CMAKE_TOOLCHAIN_FILE \
+  DUCKDB_STATIC DUCKDB_LIB_DIR DUCKDB_INCLUDE_DIR DUCKDB_DOWNLOAD_LIB \
+  RANLIB ARFLAGS CXXSTDLIB CRATE_CC_NO_DEFAULTS CC_SHELL_ESCAPED_FLAGS \
+  CMAKE_GENERATOR CMAKE_MAKE_PROGRAM CMAKE_PREFIX_PATH; do
   printf '%s=%s\n' "$variable" "${!variable-}" >>"$work/native"
 done
+# cc-rs also accepts target-specific compiler and flag overrides.
+while IFS= read -r variable; do
+  printf '%s=%s\n' "$variable" "$(printenv "$variable")" >>"$work/native"
+done < <(env | awk -F= '$1 ~ /^(CC|CXX|AR|RANLIB|CFLAGS|CXXFLAGS|ARFLAGS|CXXSTDLIB)_(x86_64|x86-64)/ || $1 ~ /^(HOST|TARGET)_(CC|CXX|AR|RANLIB|CFLAGS|CXXFLAGS|ARFLAGS|CXXSTDLIB)$/ {print $1}' | LC_ALL=C sort)
+for command in cc c++ ar ld ranlib; do
+  sha256sum "$(command -v "$command")" >>"$work/native"
+done
 for command in clang cmake mold pkg-config protoc zstd zstdmt unzstd; do "$command" --version >>"$work/native"; done
 # Bind relevant Cargo overrides without capturing registry or service secrets.
 for variable in RUSTC RUSTDOC RUSTC_WORKSPACE_WRAPPER CARGO_INCREMENTAL; do
@@ -62,17 +73,9 @@ while IFS= read -r manifest; do
 done < <(jq -r '.workspaces[].manifest' "$root/rust_hft/workspaces.json")
 recipes=$(bash "$root/.github/scripts/research-release-products.sh" recipes "${2:-all}" | jq -s .)
 locks=$("$root/.github/scripts/research-workspace-locks.sh" "$root/rust_hft")
-cat "$root/.github/scripts/build-research-release.sh" \
-  "$root/.github/scripts/build-research-recipes.sh" \
-  "$root/.github/scripts/research-cache-layout.sh" \
-  "$root/.github/scripts/research-release-products.sh" \
-  "$root/.github/scripts/research-release-products.json" \
-  "$root/.github/scripts/research-release-source-sha.sh" \
-  "$root/.github/scripts/research-workspace-locks.sh" \
-  "$root/.github/scripts/verify-research-runtime-abi.sh" \
-  "$root/.github/scripts/verify-research-runner-binaries.sh" >"$work/recipe"
-# Bind the fixed CI parser implementation alongside the build recipe.
-find "$root/.github/scripts/vendor/tomlrb/lib" -type f -name '*.rb' -print0 | LC_ALL=C sort -z | xargs -0 cat >>"$work/recipe"
+# The selected recipes below bind catalog contents to exact provenance.
+# Packaging-only catalog edits must not invalidate compatible dependencies.
+bash "$root/.github/scripts/research-cache-layout.sh" recipe-inputs >"$work/recipe"
 jq -S -n --arg compiler "$(sha256sum "$work/compiler" | awk '{print $1}')" \
   --arg native "$(sha256sum "$work/native" | awk '{print $1}')" \
   --arg flags "$(sha256sum "$work/flags" | awk '{print $1}')" \
@@ -86,6 +89,8 @@ jq -S -n --arg compiler "$(sha256sum "$work/compiler" | awk '{print $1}')" \
 if [[ -n ${GITHUB_OUTPUT:-} ]]; then printf 'cache_sha256=%s\n' "$(sha256sum "$output" | awk '{print $1}')" >>"$GITHUB_OUTPUT"; fi
 # This separate restore identity does not replace the complete v3 provenance.
 bash "$root/.github/scripts/research-cache-layout.sh" compatibility-inputs "$output" >"$work/dependency-cache-inputs.json"
+# Retain a small, public native identity summary in the producer log.
+jq -c '.native_dependencies.closures[] | {owner,duckdb:[.packages[] | select(.name=="libduckdb-sys") | {name,version,checksum,features}]}' "$work/dependency-cache-inputs.json"
 if [[ -n ${GITHUB_OUTPUT:-} ]]; then
   printf 'dependency_cache_sha256=%s\n' "$(sha256sum "$work/dependency-cache-inputs.json" | awk '{print $1}')" >>"$GITHUB_OUTPUT"
 fi

.github/scripts/ci-owner-cache.sh

@@ -130,14 +130,18 @@ begin
     "rust_hft/workspaces.json", ".github/scripts/ci-owner-cache.sh",
     ".github/scripts/test-rust-workspaces.sh", ".github/scripts/check-collector-test-presence.sh",
     ".github/workflows/ci.yml", ".github/scripts/select-rust-ci-scope.sh",
-    "deployment/aliyun/test-polymarket-raw-ops-control-plane.sh"]
+    "deployment/aliyun/test-polymarket-raw-ops-control-plane.sh", ".github/scripts/prepare-research-native-source.sh"]
   parser = Dir.glob("#{root}/.github/scripts/vendor/tomlrb/lib/**/*.rb").sort.to_h do |path|
     [path.delete_prefix("#{root}/"), file_digest(path)]
   end
   abort 'missing Cargo TOML parser' if parser.empty?
-  compatibility = {schema: 'monday.ci-owner-cache-compat.v2', compiler: compiler, target: host,
+  native_root = "#{root}/rust_hft/native/libduckdb-sys"
+  native_files = Dir.glob("#{native_root}/**/*", File::FNM_DOTMATCH).select { |path| File.file?(path) }.sort
+  abort 'native patch contains symlinks' if native_files.any? { |path| File.symlink?(path) }
+  native_patch = native_files.to_h { |path| [path.delete_prefix(native_root + '/'), file_digest(path)] }
+  compatibility = {native_patch: native_patch, native_preparation: file_digest("#{root}/.github/scripts/prepare-research-native-source.sh"), schema: 'monday.ci-owner-cache-compat.v3', compiler: compiler, target: host,
     profile: 'dev-test', native: native, flags: flags, configs: configs, profiles: profiles, owners: admitted, parser: parser}
-  prefix = "rust_hft-ci-owner-v2-#{digest(compatibility)}-"
+  prefix = "rust_hft-ci-owner-v3-#{digest(compatibility)}-"
   exact = {coverage: coverage, manifests: manifests, configs: configs,
     helpers: helpers.to_h { |path| [path, file_digest("#{root}/#{path}")] }}
   puts "compat_prefix=#{prefix}"
@@ -188,7 +192,8 @@ if [[ $mode == native-input ]]; then
       CPATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH PKG_CONFIG PKG_CONFIG_PATH \
       PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR BINDGEN_EXTRA_CLANG_ARGS \
       OPENSSL_DIR OPENSSL_LIB_DIR OPENSSL_INCLUDE_DIR OPENSSL_STATIC \
-      PROTOC PROTOC_INCLUDE CMAKE_TOOLCHAIN_FILE; do
+      PROTOC PROTOC_INCLUDE CMAKE_TOOLCHAIN_FILE \
+      DUCKDB_STATIC DUCKDB_LIB_DIR DUCKDB_INCLUDE_DIR DUCKDB_DOWNLOAD_LIB; do
       printf '%s=%s\n' "$variable" "${!variable-}"
     done
   } >"$work/native"
@@ -205,11 +210,11 @@ while IFS= read -r manifest; do
     --locked --all-features --format-version 1 >>"$work/metadata.jsonl"
 done <<<"$manifests"
 jq -se --arg root "$root/" \
-  '[.[].packages[]|select(.source == null or (.manifest_path|startswith($root)))|.name,.targets[].name]|unique|
+  '[.[].packages[]|select((.source == null or (.manifest_path|startswith($root))) and (.manifest_path != ($root+"rust_hft/native/libduckdb-sys/Cargo.toml")))|.name,.targets[].name]|unique|
    if length>0 then . else error("no repository-local compilation packages") end' \
   "$work/metadata.jsonl" >"$work/local-names.json"
 jq -s --arg root "$root/" \
-  '[.[].packages[]|select(.source != null and (.manifest_path|startswith($root)|not))|.name,.targets[].name]|unique' \
+  '[.[].packages[]|select((.source != null and (.manifest_path|startswith($root)|not)) or (.name == "libduckdb-sys" and .manifest_path == ($root+"rust_hft/native/libduckdb-sys/Cargo.toml")))|.name,.targets[].name]|unique' \
   "$work/metadata.jsonl" >"$work/external-names.json"
 ruby -rjson -rfileutils - "$root" "$work/local-names.json" "$work/external-names.json" <<'RUBY'
 root, names_file, external_file = ARGV

.github/scripts/compile-research-recipe.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Stable dependency compilation contract. Packaging and scheduling live outside it.
+set -euo pipefail
+recipe=${1:?recipe JSON required}
+jobs=${2:?Cargo jobs required}
+[[ $jobs =~ ^[1-9][0-9]*$ ]]
+manifest=$(jq -er .manifest <<<"$recipe")
+package=$(jq -er .package <<<"$recipe")
+features=$(jq -er .features <<<"$recipe")
+args=(build --manifest-path "$manifest" --target x86_64-unknown-linux-gnu --release --locked --jobs "$jobs" -p "$package")
+[[ -z $features ]] || args+=(--features "$features")
+while IFS= read -r binary; do args+=(--bin "$binary"); done < <(jq -er '.binaries[]' <<<"$recipe")
+cargo "${args[@]}" --message-format json-render-diagnostics

.github/scripts/prepare-research-native-source.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# Cache lookup binds all native bytes. Normalize only a fresh CI source tree.
+set -euo pipefail
+root=$(cd "$(dirname "$0")/../.." && pwd)
+[[ ${GITHUB_ACTIONS:-false} == true ]] || exit 0
+cd "$root"
+source=rust_hft/native/libduckdb-sys
+[[ -z $(git -c safe.directory="$root" status --porcelain --untracked-files=all -- "$source") ]] || {
+  echo 'native source must be committed and clean in CI' >&2; exit 1;
+}
+# Never hide an input change from an existing local or restored Cargo target.
+for target in rust_hft/target rust_hft/research-core/target rust_hft/data-pipelines/target; do
+  [[ ! -e $target ]] || { echo "native timestamp preparation must precede target creation: $target" >&2; exit 1; }
+done
+ruby -ropen3 - "$source" <<'RUBY'
+files, status = Open3.capture2('git', '-c', "safe.directory=#{Dir.pwd}", 'ls-files', '-z', '--', ARGV.fetch(0))
+abort 'native source inventory failed' unless status.success? && !files.empty?
+files.split("\0").each do |file|
+  abort 'native source must contain regular files' unless File.file?(file) && !File.symlink?(file)
+  File.utime(Time.at(946684800), Time.at(946684800), file)
+end
+RUBY

.github/scripts/research-cache-layout.sh

@@ -38,6 +38,16 @@ validate_recipes() {
       (.binaries|type == "array" and length>0 and all(.[];name)))' "$1" >/dev/null
 }
 case ${1:?command required} in
+  recipe-inputs)
+    # Bind compiler orchestration and cleanup, independently of product selection.
+    # Exact selected recipes remain in compilation inputs and artifact admission.
+    for script in build-research-release.sh build-research-recipes.sh capture-research-build-inputs.sh research-cache-layout.sh \
+      research-release-products.sh research-release-source-sha.sh research-workspace-locks.sh \
+      verify-research-runtime-abi.sh verify-research-runner-binaries.sh research-native-dependency-inputs.rb compile-research-recipe.sh prepare-research-native-source.sh; do
+      cat "$root/.github/scripts/$script"
+    done
+    find "$root/.github/scripts/vendor/tomlrb/lib" -type f -name '*.rb' -print0 | LC_ALL=C sort -z | xargs -0 cat
+    ;;
   compatibility-inputs)
     # Keep local dependency choices in provenance and Cargo's exact cache suffix.
     # The restore prefix still binds every admitted compiler, native and profile input.
@@ -124,9 +134,12 @@ rescue StandardError => error
   exit 1
 end
 RUBY
-    jq -Se --slurpfile owner_profiles "$work/owner-profiles.json" '
-      {schema:"monday.dependency-cache-compat.v3",target,profile,compiler,native,flags,
-        builder_image,recipe,owner_profiles:$owner_profiles[0]}' "$work/validated.json"
+    ruby "$root/.github/scripts/research-native-dependency-inputs.rb" "$root" "$work/cache-recipes.json" >"$work/native-dependencies.json"
+    dependency_recipe=$(cat "$root/.github/scripts/compile-research-recipe.sh" "$root/.github/scripts/research-cache-layout.sh" \
+      "$root/.github/scripts/research-native-dependency-inputs.rb" "$root/.github/scripts/prepare-research-native-source.sh" | sha256sum | awk '{print $1}')
+    jq -Se --arg dependency_recipe "$dependency_recipe" --slurpfile native_dependencies "$work/native-dependencies.json" --slurpfile owner_profiles "$work/owner-profiles.json" '
+      {schema:"monday.dependency-cache-compat.v4",target,profile,compiler,native,flags,
+        builder_image,dependency_recipe:$dependency_recipe,owner_profiles:$owner_profiles[0],native_dependencies:$native_dependencies[0]}' "$work/validated.json"
     ;;
   manifest-inputs)
     # Include local path/patch/default-feature manifests beyond recipe roots.
@@ -198,8 +211,10 @@ RUBY
       fi
       cargo metadata --manifest-path "$root/rust_hft/$manifest" --locked --format-version 1 "${metadata_args[@]}" >>"$work/local.jsonl"
     done <<<"$recipes"
-    jq -s --arg root "$root/" '[.[].packages[]|select(.source == null or (.manifest_path|startswith($root)))|.name, .targets[].name]|unique|if length>0 then . else error("no local compilation packages") end' "$work/local.jsonl" >"$work/names.json"
-    jq -s --arg root "$root/" '[.[].packages[]|select(.source != null and (.manifest_path|startswith($root)|not))|.name, .targets[].name]|unique' "$work/local.jsonl" >"$work/external.json"
+    # The reviewed native patch is a dependency. Its complete source enters the
+    # native compatibility identity; application path packages remain excluded.
+    jq -s --arg root "$root/" '[.[].packages[]|select((.source == null or (.manifest_path|startswith($root))) and (.manifest_path != ($root+"rust_hft/native/libduckdb-sys/Cargo.toml")))|.name, .targets[].name]|unique|if length>0 then . else error("no local compilation packages") end' "$work/local.jsonl" >"$work/names.json"
+    jq -s --arg root "$root/" '[.[].packages[]|select((.source != null and (.manifest_path|startswith($root)|not)) or (.name == "libduckdb-sys" and .manifest_path == ($root+"rust_hft/native/libduckdb-sys/Cargo.toml")))|.name, .targets[].name]|unique' "$work/local.jsonl" >"$work/external.json"
     ruby -rjson -rfileutils - "$root" "$work/names.json" "$work/external.json" "$work/cache-recipes.json" <<'RUBY'
 root, names_file, external_file, recipes_file = ARGV
 names = JSON.parse(File.read(names_file)).flat_map { |n| [n, n.tr('-', '_'), "lib#{n.tr('-', '_')}"] }.uniq