diff --git a/.github/workflows/build-aimrocks.yml b/.github/workflows/build-aimrocks.yml new file mode 100644 index 000000000000..51b6c676e860 --- /dev/null +++ b/.github/workflows/build-aimrocks.yml @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# This workflow is based on the `deps`/`rocksdb`/`wheels` targets of +# https://github.com/aimhubio/aimrocks/blob/v0.5.2/Dockerfile and the +# docker/build-*.sh scripts it runs. +name: Build aimrocks wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/aimrocks.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-aimrocks.yml' + - 'docs/packages/aimrocks.yaml' + push: + branches: [main] + paths: + - '.github/workflows/build-aimrocks.yml' + - 'docs/packages/aimrocks.yaml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + # Versions upstream's own docker/build-*.sh pin. + ZLIB_VERSION: '1.2.11' + BZIP2_VERSION: '1.0.8' + LZ4_VERSION: '1.9.3' + ZSTD_VERSION: '1.1.3' + SNAPPY_VERSION: '1.1.8' + ROCKSDB_VERSION: '6.29.fb' + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: aimrocks + version: ${{ inputs.version }} + + build_wheels: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Build aimrocks ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + python: + - "cp312" + - "cp313" + - "cp314" + - "cp314t" + + env: + AIMROCKS_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout aimrocks ${{ env.AIMROCKS_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: aimhubio/aimrocks + ref: v${{ env.AIMROCKS_VERSION }} + persist-credentials: false + + - name: Build wheels + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + with: + output-dir: wheelhouse/ + only: ${{ matrix.python }}-manylinux_riscv64 + env: + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} + CIBW_ENVIRONMENT_PASS_LINUX: >- + ZLIB_VERSION BZIP2_VERSION LZ4_VERSION ZSTD_VERSION SNAPPY_VERSION + ROCKSDB_VERSION + # Reproduces upstream's docker/build-{zlib,bzip2,lz4,zstd,snappy,rocksdb}.sh: + # each dep installs into /opt/deps with its default (mostly-static) `make + # install`/cmake, then the `rm -f *.so*` before the rocksdb build (matching + # upstream's own `rm -rf lib/lib*.so*` in build-rocksdb.sh) forces rocksdb to + # link every codec in statically, so the only shared object produced is + # librocksdb.so itself. PORTABLE=1 (upstream's own flag, used unconditionally + # there) disables -march=native and every x86 SSE4.2/PCLMUL/AVX2 probe in + # rocksdb's build_detect_platform; util/crc32c.cc's hardware paths are gated on + # HAVE_SSE42/HAVE_ARM64_CRC/HAVE_POWER8, none of which get defined here, so it + # falls back to the portable table-based CRC32C upstream already ships + # non-x86 wheels (aarch64) with. DISABLE_WARNING_AS_ERROR=1 is rocksdb's own + # Makefile knob (gated by `ifndef DISABLE_WARNING_AS_ERROR`) for a compiler + # newer than 6.29.fb's code anticipated: manylinux_2_39_riscv64's GCC 14.3.1 + # flags `return std::move(io_s);` in db/db_impl/db_impl.cc as + # -Werror=redundant-move, which upstream's own (older) build image never hit. + CIBW_BEFORE_ALL_LINUX: >- + mkdir -p /opt/deps/lib /opt/deps/include && + curl -sSLf "https://www.zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz" | tar -xz -C /tmp && + (cd /tmp/zlib-${ZLIB_VERSION} && ./configure && make CFLAGS=-fPIC CXXFLAGS=-fPIC && make install prefix=/opt/deps) && + curl -sSLf "https://sourceware.org/pub/bzip2/bzip2-${BZIP2_VERSION}.tar.gz" | tar -xz -C /tmp && + (cd /tmp/bzip2-${BZIP2_VERSION} && make CFLAGS=-fPIC CXXFLAGS=-fPIC && make install PREFIX=/opt/deps) && + curl -sSLf "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz" | tar -xz -C /tmp && + (cd /tmp/lz4-${LZ4_VERSION} && make CFLAGS=-fPIC CXXFLAGS=-fPIC && make PREFIX=/opt/deps install) && + curl -sSLf "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz" | tar -xz -C /tmp && + (cd /tmp/zstd-${ZSTD_VERSION} && make CFLAGS=-fPIC CXXFLAGS=-fPIC && make PREFIX=/opt/deps install) && + curl -sSLf "https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz" | tar -xz -C /tmp && + cmake -S /tmp/snappy-${SNAPPY_VERSION} -B /tmp/snappy-${SNAPPY_VERSION}/build + -DCMAKE_INSTALL_PREFIX=/opt/deps -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 && + cmake --build /tmp/snappy-${SNAPPY_VERSION}/build --parallel --target install && + rm -f /opt/deps/lib/lib*.so* && + curl -sSLf "https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz" | tar -xz -C /tmp && + LIBRARY_PATH=/opt/deps/lib PORTABLE=1 make -C /tmp/rocksdb-${ROCKSDB_VERSION} + shared_lib PLATFORM_SHARED_VERSIONED=false EXTRA_CXXFLAGS="-fPIC -I/opt/deps/include" + EXTRA_CFLAGS=-fPIC USE_RTTI=0 DEBUG_LEVEL=0 DISABLE_WARNING_AS_ERROR=1 -j4 && + strip --strip-debug /tmp/rocksdb-${ROCKSDB_VERSION}/librocksdb.so && + cp /tmp/rocksdb-${ROCKSDB_VERSION}/librocksdb.so /opt/deps/lib/ && + PORTABLE=1 make -C /tmp/rocksdb-${ROCKSDB_VERSION} PREFIX=/opt/deps DEBUG_LEVEL=0 install-headers && + cp /tmp/zlib-${ZLIB_VERSION}/README {project}/LICENSE.zlib && + cp /tmp/bzip2-${BZIP2_VERSION}/LICENSE {project}/LICENSE.bzip2 && + cp /tmp/lz4-${LZ4_VERSION}/lib/LICENSE {project}/LICENSE.lz4 && + cp /tmp/zstd-${ZSTD_VERSION}/LICENSE {project}/LICENSE.zstd && + cp /tmp/snappy-${SNAPPY_VERSION}/COPYING {project}/LICENSE.snappy && + cp /tmp/rocksdb-${ROCKSDB_VERSION}/LICENSE.Apache {project}/LICENSE.rocksdb + CIBW_ENVIRONMENT: AIM_DEP_DIR=/opt/deps + CIBW_REPAIR_WHEEL_COMMAND: auditwheel repair --strip -w {dest_dir} {wheel} + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_SOURCES: tests + CIBW_TEST_COMMAND: pytest tests + + - name: Check the extension and licences made it into the wheel + run: | + python3 - wheelhouse/*.whl <<'EOF' + import sys, zipfile + for whl in sys.argv[1:]: + names = zipfile.ZipFile(whl).namelist() + assert any(n.startswith("aimrocks/lib_rocksdb") and ".so" in n for n in names), whl + assert any(n.endswith("licenses/LICENSE") for n in names), whl + for dep in ("zlib", "bzip2", "lz4", "zstd", "snappy", "rocksdb"): + assert any(n.endswith(f"licenses/LICENSE.{dep}") for n in names), (whl, dep) + print(whl, "ok") + EOF + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: aimrocks-${{ env.AIMROCKS_VERSION }}-${{ matrix.python }}-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + publish: + name: Publish aimrocks ${{ matrix.version }} + needs: [setup, build_wheels] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: aimrocks-${{ matrix.version }}-*-manylinux_riscv64 diff --git a/docs/packages/aimrocks.yaml b/docs/packages/aimrocks.yaml new file mode 100644 index 000000000000..ba449ab81244 --- /dev/null +++ b/docs/packages/aimrocks.yaml @@ -0,0 +1,5 @@ +package-name: aimrocks +source-code: https://github.com/aimhubio/aimrocks +license: Apache License 2.0 +versions: +- version: 0.5.2