Build SQLCipher from sqlcipher-src 419.0.6's separate libtomcrypt sources

Author: LucaCappelletti94

fuzz/Cargo.lock

@@ -8,11 +8,17 @@ use std::{
     process::Command,
 };
 
+/// One compiler invocation of a side, its objects merge before the rename.
+struct Unit<'a> {
+    files: &'a [PathBuf],
+    includes: &'a [PathBuf],
+    defines: &'a [(&'a str, Option<&'a str>)],
+    forced_include: Option<&'a str>,
+}
+
 struct Side<'a> {
     prefix: &'a str,
-    source: PathBuf,
-    include: &'a Path,
-    defines: &'a [(&'a str, Option<&'a str>)],
+    units: &'a [Unit<'a>],
 }
 
 fn main() {
@@ -22,29 +28,60 @@ fn main() {
         "the symbol renaming needs ELF objects and GNU-compatible nm and objcopy, so the fuzz crate builds on Linux only"
     );
     let out = PathBuf::from(env::var_os("OUT_DIR").expect("cargo sets OUT_DIR"));
-    let mc_dir = sqlite3mc_src::source_dir();
-    let sc_dir = sqlcipher_src::source_dir();
+    let crate_dir =
+        PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("cargo sets CARGO_MANIFEST_DIR"));
+    let mc_dir = sqlite3mc_src::source_dir().to_path_buf();
+    let sc_dir = sqlcipher_src::source_dir().to_path_buf();
+    let ltc_dir = sc_dir.join(sqlcipher_src::LIBTOMCRYPT_INCLUDE_DIR);
     let common = [
         ("SQLITE_TEMP_STORE", Some("2")),
         ("SQLITE_ENABLE_DBPAGE_VTAB", None),
         ("SQLITE_DEFAULT_MEMSTATUS", Some("0")),
     ];
-    let mc_defines = [common.as_slice(), &[("SQLITE_CORE", None)]].concat();
+    let mc_defines: Vec<(&str, Option<&str>)> =
+        [common.as_slice(), &[("SQLITE_CORE", None)]].concat();
+    let ltc_files: Vec<PathBuf> = std::iter::once("sqlcipher-entropy.c")
+        .map(|name| crate_dir.join(name))
+        .chain(
+            sqlcipher_src::LIBTOMCRYPT_SOURCES
+                .iter()
+                .map(|source| sc_dir.join(source)),
+        )
+        .collect();
+    let mc_files = [mc_dir.join(sqlite3mc_src::SOURCE_FILE)];
+    let mc_includes = [mc_dir];
+    let wrapper_files = [crate_dir.join("sqlcipher-wrapper.c")];
+    let wrapper_includes = [sc_dir, ltc_dir.clone()];
+    let ltc_includes = [ltc_dir, crate_dir];
     build(
         &Side {
             prefix: "mc",
-            source: mc_dir.join(sqlite3mc_src::SOURCE_FILE),
-            include: mc_dir,
-            defines: &mc_defines,
+            units: &[Unit {
+                files: &mc_files,
+                includes: &mc_includes,
+                defines: &mc_defines,
+                forced_include: None,
+            }],
         },
         &out,
     );
     build(
         &Side {
             prefix: "sc",
-            source: sc_dir.join(sqlcipher_src::WASM_SOURCE_FILE),
-            include: sc_dir,
-            defines: &common,
+            units: &[
+                Unit {
+                    files: &wrapper_files,
+                    includes: &wrapper_includes,
+                    defines: &common,
+                    forced_include: None,
+                },
+                Unit {
+                    files: &ltc_files,
+                    includes: &ltc_includes,
+                    defines: &[("LTC_SOURCE", None)],
+                    forced_include: Some("sqlcipher-ltc.h"),
+                },
+            ],
         },
         &out,
     );
@@ -53,38 +90,80 @@ fn main() {
         println!("cargo:rustc-link-lib={lib}");
     }
     println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-changed=sqlcipher-wrapper.c");
+    println!("cargo:rerun-if-changed=sqlcipher-entropy.c");
+    println!("cargo:rerun-if-changed=sqlcipher-ltc.h");
     println!("cargo:rerun-if-env-changed=NM");
     println!("cargo:rerun-if-env-changed=OBJCOPY");
+    println!("cargo:rerun-if-env-changed=LD");
+    println!("cargo:rerun-if-env-changed=AR");
 }
 
 fn build(side: &Side<'_>, out: &Path) {
-    println!("cargo:rerun-if-changed={}", side.include.display());
-    let mut cc = cc::Build::new();
-    cc.file(&side.source).include(side.include).warnings(false);
-    for (name, value) in side.defines {
-        cc.define(name, *value);
-    }
-    instrument(&mut cc);
-    let objects = cc.compile_intermediates();
-    let [object] = objects.as_slice() else {
-        panic!(
+    let mut objects = Vec::new();
+    for unit in side.units {
+        for file in unit.files {
+            println!("cargo:rerun-if-changed={}", file.display());
+        }
+        let mut cc = cc::Build::new();
+        for file in unit.files {
+            cc.file(file);
+        }
+        for dir in unit.includes {
+            cc.include(dir);
+        }
+        cc.warnings(false);
+        for (name, value) in unit.defines {
+            cc.define(name, *value);
+        }
+        if let Some(header) = unit.forced_include {
+            cc.flag("-include").flag(header);
+        }
+        instrument(&mut cc);
+        let unit_objects = cc.compile_intermediates();
+        assert_eq!(
+            unit_objects.len(),
+            unit.files.len(),
             "{} compiled to {} objects",
-            side.source.display(),
-            objects.len()
+            unit.files[0].display(),
+            unit_objects.len()
         );
+        objects.extend(unit_objects);
+    }
+
+    let combined = if objects.len() == 1 {
+        objects.remove(0)
+    } else {
+        // Clang's .group sections break the final link once a partial link merges them.
+        let linked: Vec<PathBuf> = objects
+            .iter()
+            .map(|object| strip_group_sections(object))
+            .collect();
+        let combined = out.join(format!("{}.combined.o", side.prefix));
+        // The per-global asan sections survive a bfd partial link, while a gold one
+        // drops most of them and ASan then never registers the side's globals.
+        run(Command::new(tool("LD", "ld"))
+            .arg("-r")
+            .arg("-o")
+            .arg(&combined)
+            .args(&linked));
+        combined
     };
 
     let map = out.join(format!("{}.redefine", side.prefix));
-    fs::write(&map, redefinitions(object, side.prefix)).expect("OUT_DIR is writable");
+    fs::write(&map, redefinitions(&combined, side.prefix)).expect("OUT_DIR is writable");
     let renamed = out.join(format!("{}.o", side.prefix));
     run(Command::new(tool("OBJCOPY", "objcopy"))
         .arg(format!("--redefine-syms={}", map.display()))
-        .arg(object)
+        .arg(&combined)
         .arg(&renamed));
 
     let archive = out.join(format!("lib{}.a", side.prefix));
     let _ = fs::remove_file(&archive);
-    run(cc.get_archiver().arg("crs").arg(&archive).arg(&renamed));
+    run(Command::new(tool("AR", "ar"))
+        .arg("crs")
+        .arg(&archive)
+        .arg(&renamed));
     println!("cargo:rustc-link-lib=static={}", side.prefix);
 }
 
@@ -118,6 +197,17 @@ fn redefinitions(object: &Path, prefix: &str) -> String {
         })
 }
 
+/// Returns a copy of `object` without its `.group` section.
+fn strip_group_sections(object: &Path) -> PathBuf {
+    let stem = object.file_stem().expect("object has a stem");
+    let stripped = object.with_file_name(format!("{}.ng.o", stem.to_string_lossy()));
+    run(Command::new(tool("OBJCOPY", "objcopy"))
+        .arg("--remove-section=.group")
+        .arg(object)
+        .arg(&stripped));
+    stripped
+}
+
 fn tool(var: &str, default: &str) -> String {
     env::var(var).unwrap_or_else(|_| default.to_owned())
 }

fuzz/build.rs

@@ -0,0 +1,24 @@
+/* SQLCipher's entropy hook, giving its LibTomCrypt RNG the harness's deterministic getentropy. */
+#include <stddef.h>
+#include <stdlib.h>
+
+#include "sqlcipher-ltc.h"
+#include <tomcrypt.h>
+
+/* Declared here, the fuzz harness provides it at link time. */
+int getentropy(void *buffer, size_t len);
+int sqlcipher_extra_init(const char *arg);
+
+long sqlcipher_wasm_no_clock(void) { abort(); }
+
+/* Fails closed so rng_get_bytes never falls through to the clock. */
+static unsigned long sqlcipher_wasm_rng(unsigned char *out, unsigned long len, void (*callback)(void)) {
+  (void)callback;
+  if (getentropy(out, len) != 0) abort();
+  return len;
+}
+
+int sqlcipher_wasm_extra_init(const char *arg) {
+  ltc_rng = sqlcipher_wasm_rng;
+  return sqlcipher_extra_init(arg);
+}

fuzz/sqlcipher-entropy.c

@@ -0,0 +1,29 @@
+/* SQLCipher's one LibTomCrypt configuration, shared by every SQLCipher and LibTomCrypt unit so struct layouts agree. */
+#ifndef SQLCIPHER_LTC_H
+#define SQLCIPHER_LTC_H
+
+#define LTC_NOTHING
+#define LTC_RIJNDAEL
+#define LTC_CBC_MODE
+#define LTC_SHA1
+#define LTC_SHA256
+#define LTC_SHA512
+#define LTC_HASH_HELPERS
+#define LTC_HMAC
+#define LTC_FORTUNA
+#define LTC_PKCS_5
+#define LTC_RNG_GET_BYTES
+#define LTC_PRNG_ENABLE_LTC_RNG
+#define LTC_NO_TEST
+#define LTC_NO_FILE
+#define LTC_NO_ASM
+#define LTC_NO_MATH
+#define LTC_CLEAN_STACK
+/* Bad arguments return CRYPT_INVALID_ARG instead of raising a signal, which the fuzzer would count as a crash. */
+#define ARGTYPE 4
+/* rng_get_bytes always ends in a clock-jitter generator, so reaching it must abort. */
+#define XCLOCK sqlcipher_wasm_no_clock
+#define XCLOCKS_PER_SEC 1
+long sqlcipher_wasm_no_clock(void);
+
+#endif

fuzz/sqlcipher-ltc.h

@@ -0,0 +1,17 @@
+/* SQLCipher with its LibTomCrypt provider, whose sources build as separate units with the same configuration. */
+#define SQLITE_HAS_CODEC 1
+#define SQLCIPHER_CRYPTO_LIBTOMCRYPT 1
+/* Installs the entropy hook before SQLCipher's own init runs. */
+#define SQLITE_EXTRA_INIT sqlcipher_wasm_extra_init
+#define SQLITE_EXTRA_SHUTDOWN sqlcipher_extra_shutdown
+/* SQLCipher's log needs a timestamp and a device the harness does not keep. */
+#define SQLCIPHER_OMIT_LOG 1
+#define SQLCIPHER_OMIT_LOG_DEVICE 1
+#define OMIT_MEMLOCK 1
+/* SQLCipher refuses SQLITE_THREADSAFE=0, and the harness runs single-threaded, so no-op mutexes suffice. */
+#undef SQLITE_THREADSAFE
+#define SQLITE_THREADSAFE 1
+#define SQLITE_MUTEX_NOOP 1
+
+#include "sqlcipher-ltc.h"
+#include "sqlcipher.c"

fuzz/sqlcipher-wrapper.c

@@ -1,5 +1,5 @@
 //! Replaces libc's `getentropy`, the call SQLCipher's RNG makes directly (`sqlcipher_wasm_rng` in
-//! `WASM_SOURCE_FILE`), with a deterministic stream reset once per [`run`](crate::run::run) call, so
+//! `sqlcipher-entropy.c`), with a deterministic stream reset once per [`run`](crate::run::run) call, so
 //! the same [`Case`](crate::Case) produces the same salts and IVs on every repeat, not just the same
 //! workload. SQLite3MC's own RNG reads `/dev/urandom` first and falls back to `getentropy` only if
 //! that open fails, so this does not make SQLite3MC's own writes deterministic; it is enough for any

fuzz/src/entropy.rs

@@ -13,6 +13,14 @@ mod ffi;
 mod input;
 mod run;
 pub mod seedgen;
+/// The `ASan` default `detect_odr_violation` level (2 since clang 18) reports the two
+/// renamed copies of the SQLite amalgamation as a violation, so the harness turns the
+/// check off. `ASan` reads its default options from the weak symbol of this name, a
+/// function returning the options string.
+#[no_mangle]
+const extern "C" fn __asan_default_options() -> *const u8 {
+    c"detect_odr_violation=0".as_ptr().cast::<u8>()
+}
 
 pub use input::{
     Algorithm, AttachCase, BackupCase, Case, Checkpoint, Cipher, Config, Key, NativeCase,