diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe0f5cd67..d2e771431 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,6 +54,10 @@ jobs: run: | echo EXTRA_ZEPHYR_MODULES="$(pwd)/$MODULE_PATH" >> $GITHUB_ENV + - name: Fetch the Arduino API + run: | + west blobs --auto-accept fetch arduino-api + - name: Build fade run: | west build -p -b arduino_nano_33_ble//sense $MODULE_PATH/samples/fade diff --git a/.github/workflows/commit_check.yml b/.github/workflows/commit_check.yml index da0408a74..19424d5aa 100644 --- a/.github/workflows/commit_check.yml +++ b/.github/workflows/commit_check.yml @@ -19,7 +19,7 @@ jobs: with: pattern: '^[^!]+: [A-Za-z]+.+ .+$' flags: 'gm' - error: 'Commit subject line must match the following pattern: : ' + error: 'Commit subject line / PR title must match the following pattern: : ' excludeTitle: 'false' excludeDescription: 'true' checkAllCommitMessages: 'true' @@ -29,7 +29,7 @@ jobs: with: pattern: '^[^#].{10,80}$' error: 'Commit subject line maximum line length of 80 characters is exceeded.' - excludeTitle: 'false' + excludeTitle: 'true' excludeDescription: 'true' checkAllCommitMessages: 'true' accessToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml index c0726a87d..0d29ea846 100644 --- a/.github/workflows/format_check.yml +++ b/.github/workflows/format_check.yml @@ -35,7 +35,7 @@ jobs: cores/arduino/api/** loader/llext_exports.c loader/blobs/4343WA1_*.c - loader/blobs/wifi_nvram_image.h + loader/blobs/wifi_nvram_image_data.h libraries/examples/** libraries/extras/** libraries/ea_malloc/** diff --git a/.github/workflows/package_core.yml b/.github/workflows/package_core.yml index 4c64c8b37..567077138 100644 --- a/.github/workflows/package_core.yml +++ b/.github/workflows/package_core.yml @@ -53,10 +53,13 @@ jobs: ARTIFACTS: ${{ env.ARTIFACTS }} # List of architectures the artifacts will be built for SUB_ARCHES: ${{ env.SUB_ARCHES }} + # URL of the latest ci-builds index for this branch, if any + CI_BUILDS_URL: ${{ env.CI_BUILDS_URL }} steps: - name: Install OS dependencies working-directory: /opt + timeout-minutes: 5 # early error on Github network issues run: | sudo apt-get remove --purge man-db -y # skips the mandb triggers sudo apt-get update @@ -69,10 +72,8 @@ jobs: persist-credentials: false fetch-tags: true # needed for get_core_version.sh - - name: Initialize Zephyr environment + - name: Identify current version and board set run: | - # Install Zephyr as a core user would, but faster - yes | ./extra/bootstrap.sh -o=--filter=tree:0 # Set up basic environment for later steps echo "CORE_HASH=$(git describe --always $HEAD_REF)" >> "$GITHUB_ENV" echo "ALL_BOARD_DATA=$(extra/get_board_details.sh | jq -c 'sort_by(.variant)')" >> "$GITHUB_ENV" @@ -88,15 +89,48 @@ jobs: [[ -n "$BRANCH" ]] || { echo "::error::No branch found containing tag ${{ github.ref_name }}"; exit 1; } echo "CORE_BRANCH=${BRANCH}" >> "$GITHUB_ENV" else - echo "CORE_BRANCH=${{ github.head_ref || github.ref_name }}" >> "$GITHUB_ENV" + echo "CORE_BRANCH=${{ github.base_ref || github.ref_name }}" >> "$GITHUB_ENV" fi echo "ALL_BOARD_FQBNS=$(jq -c 'map((. + {link_mode: "static"}), (. + {link_mode: "dynamic"}))' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV" echo "ARTIFACTS=$(jq -c '["zephyr"] + (map(.artifact) | unique)' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV" echo "SUB_ARCHES=$(jq -c 'map(.subarch) | unique' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV" + + - name: Cache west modules and blobs + uses: actions/cache@v5 + with: + key: west-cache-${{ hashFiles('west.yml', 'boards.txt') }} + restore-keys: | + west-blobs- + path: ${{ runner.temp }}/west-cache + + - name: Initialize Zephyr environment + timeout-minutes: 10 # early error on Github network issues + env: + WEST_CACHE: ${{ runner.temp }}/west-cache + run: | + # Install Zephyr as a core user would, but faster + yes | ./extra/bootstrap.sh -o=--filter=tree:0 # Archive the build environment for later jobs (cd && tar cphf - .cmake work zephyr-sdk-* | zstd > build-env.tar.zstd) - # The API is only needed for packaging, not for building, so it's separate - tar cphf - cores/arduino/api | zstd > arduino-api.tar.zstd + + - name: Get ci-builds package index link + if: ${{ github.repository == 'arduino/ArduinoCore-zephyr' }} + run: | + # for pull requests and pushes to the official Arduino repo, use the latest ci-built core as a reference + CI_BUILDS_URL="https://arduino.github.io/core-ci-builds/package_arduino_zephyr_${CORE_BRANCH}_ci_index.json" + if ! wget -q --method=HEAD "$CI_BUILDS_URL" >& /dev/null ; then + CLOSEST_BRANCH=$(git branch -r | grep -vE "HEAD|/$(git branch --show-current)$" | while read -r b; do \ + echo "$(git rev-list --count $b..HEAD) $b" ; \ + done | sort -n | head -n 1 | awk '{print $2}' | cut -d / -f 2) + [ -z "$CLOSEST_BRANCH" ] && CLOSEST_BRANCH="main" + echo "::warning::No ci-builds index found for branch $CORE_BRANCH, using $CLOSEST_BRANCH" + CI_BUILDS_URL="https://arduino.github.io/core-ci-builds/package_arduino_zephyr_${CLOSEST_BRANCH}_ci_index.json" + if ! wget -q --method=HEAD "$CI_BUILDS_URL" >& /dev/null ; then + echo "::warning::No ci-builds index found for branch $CLOSEST_BRANCH, using main" + CI_BUILDS_URL="https://arduino.github.io/core-ci-builds/package_arduino_zephyr_main_ci_index.json" + fi + fi + echo "CI_BUILDS_URL=$CI_BUILDS_URL" >> "$GITHUB_ENV" - name: Download test data run: | @@ -117,14 +151,6 @@ jobs: retention-days: 1 archive: false - - name: Archive API snapshot - uses: actions/upload-artifact@v7 - with: - name: arduino-api - path: arduino-api.tar.zstd - retention-days: 1 - archive: false - # Extract the build environment and compile the loader project for each board @@ -152,11 +178,14 @@ jobs: steps: - uses: actions/download-artifact@v8 + timeout-minutes: 10 # early error on Github network issues with: path: /home/runner name: build-env.tar.zstd - name: Restore build environment + id: restore-env + timeout-minutes: 5 # early error on Github network issues run: | sudo apt-get remove --purge man-db -y # skips the mandb triggers sudo apt-get update @@ -208,7 +237,7 @@ jobs: echo "::endgroup::" - name: Package board artifacts - if: ${{ !cancelled() }} + if: ${{ steps.restore-env.outcome == 'success' && !cancelled() }} run: | # add the config file to the report files, list them for later archiving cp firmwares/zephyr-${{ matrix.variant }}.config $REPORT.config @@ -223,11 +252,12 @@ jobs: firmwares/*${{ matrix.variant }}* \ variants/${{ matrix.variant }}/ \ boards.local_${{ matrix.variant }}.txt \ + cores/arduino/api/ \ ${FULL_BUILD_DIR} \ | zstd > binaries-${ARTIFACT_TAG}.tar.zstd - name: Archive board binaries - if: ${{ !cancelled() }} + if: ${{ steps.restore-env.outcome == 'success' && !cancelled() }} uses: actions/upload-artifact@v7 with: # prefix the name with 'failed-' if the build failed, so that it is not removed @@ -237,7 +267,7 @@ jobs: archive: false - name: Archive build reports - if: ${{ !cancelled() }} + if: ${{ steps.restore-env.outcome == 'success' && !cancelled() }} uses: actions/upload-artifact@v7 with: name: build-report-${{ env.ARTIFACT_TAG }} @@ -275,12 +305,6 @@ jobs: persist-credentials: false fetch-tags: true - # Get the API snapshot - - uses: actions/download-artifact@v8 - with: - path: . - name: arduino-api.tar.zstd - # Get all built binaries - uses: actions/download-artifact@v8 with: @@ -290,9 +314,8 @@ jobs: - name: Package core run: | - # Extract the API snapshot and board binaries in place - rm -f cores/arduino/api # remove broken symlink - tar --use-compress-program=unzstd -xpf arduino-api.tar.zstd + # Extract the board binaries in place + # (each will also include the same copy of the API) for f in binaries-*.tar.zstd ; do tar --use-compress-program=unzstd -xpf $f done @@ -301,22 +324,113 @@ jobs: ./extra/package_core.sh ${{ matrix.artifact }} ${CORE_TAG} distrib/${CORE_ARTIFACT} - uses: actions/upload-artifact@v7 + id: upload-core if: ${{ success() || failure() }} with: name: ${{ env.CORE_ARTIFACT }} path: distrib/${{ env.CORE_ARTIFACT }} archive: false + - name: Archive artifact ID + run: | + echo '${{ steps.upload-core.outputs.artifact-id }}' > ${{ matrix.artifact }}.id + + - uses: actions/upload-artifact@v7 + with: + name: artifact-id-${{ matrix.artifact }} + path: ${{ matrix.artifact }}.id + + + + # Prepare this PR's temporary package index JSON by merging the newly + # built core packages with the latest package index from ci-builds. + + package-index: + name: Package index URL for offline tests + runs-on: ubuntu-latest + needs: + - build-env + - package-core + if: ${{ github.event_name == 'pull_request' && needs.build-env.outputs.CI_BUILDS_URL != '' }} + env: + CORE_HASH: ${{ needs.build-env.outputs.CORE_HASH }} + CORE_BRANCH: ${{ needs.build-env.outputs.CORE_BRANCH }} + ARTIFACTS: ${{ needs.build-env.outputs.ARTIFACTS }} + CI_BUILDS_URL: ${{ needs.build-env.outputs.CI_BUILDS_URL }} + steps: + + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + fetch-tags: true + + - uses: actions/download-artifact@v8 + with: + pattern: ArduinoCore-*.tar.bz2 + path: . + merge-multiple: true + + - uses: actions/download-artifact@v8 + with: + pattern: artifact-id-* + path: . + merge-multiple: true + + - name: Prepare PR package index + run: | + wget ${CI_BUILDS_URL} -O ci_index.json + # clear other platforms from the index + jq '.packages[0].platforms = []' ci_index.json > ci_index.tmp && mv ci_index.tmp ci_index.json + + mkdir -p snippets/platforms/ + jq -cr '.[]' <<< ${ARTIFACTS} | while read -r artifact; do + [ $artifact == "zephyr" ] && continue + ARTIFACT_FILE=ArduinoCore-${artifact}-${CORE_HASH}.tar.bz2 + PACKAGE_JSON=snippets/platforms/${artifact}.json + ./extra/gen_package_index_json.sh ${artifact} ${ARTIFACT_FILE} ${PACKAGE_JSON} + # replace the download URL with this PR's artifact link + ARTIFACT_ID=$(cat ${artifact}.id) + ARTIFACT_URL="https://api.github.com/repos/${{ github.repository }}/actions/artifacts/${ARTIFACT_ID}/zip" + sed -i -e "s!\"url\":.*!\"url\": \"${ARTIFACT_URL}\",!" ${PACKAGE_JSON} + # append the new snippet to the index + jq -s '.[0].packages[0].platforms += .[1].packages[0].platforms | .[0]' \ + ci_index.json ${PACKAGE_JSON} > ci_index.tmp && mv ci_index.tmp ci_index.json + done + PACKAGE_INDEX_FILE="package_arduino_zephyr_pr_${{ github.event.pull_request.number }}_ci_index.json" + echo "PACKAGE_INDEX_FILE=${PACKAGE_INDEX_FILE}" >> $GITHUB_ENV + mv ci_index.json ${PACKAGE_INDEX_FILE} + + - name: Archive PR package index + id: package-index-json + uses: actions/upload-artifact@v7 + with: + name: ${{ env.PACKAGE_INDEX_FILE }} + path: ${{ env.PACKAGE_INDEX_FILE }} + archive: false + + - name: Display final URL in the summary + run: | + ARTIFACT_ID=${{ steps.package-index-json.outputs.artifact-id }} + PACKAGE_INDEX_URL="https://api.github.com/repos/${{ github.repository }}/actions/artifacts/${ARTIFACT_ID}/zip" + PACKAGE_INDEX_URL="${PACKAGE_INDEX_URL}#${{ env.PACKAGE_INDEX_FILE }}" + echo "[${PACKAGE_INDEX_URL}](${PACKAGE_INDEX_URL})" >> $GITHUB_STEP_SUMMARY + - # After generating the final cores, the API and per-board binary archives are no longer needed. - # The build environment archive can also be removed. + # After generating the final cores, the following temporary artifacts are + # no longer used and can be removed: + # + # - the Zephyr build environment + # - the API and per-board binaries + # - the artifact download URLs cleanup-build: name: Clean up intermediates runs-on: ubuntu-latest needs: - package-core + - package-index steps: - uses: geekyeggo/delete-artifact@v6 @@ -325,6 +439,7 @@ jobs: arduino-api.tar.zstd binaries-* build-env.tar.zstd + artifact-id-* failOnError: false @@ -350,6 +465,7 @@ jobs: FQBN: arduino:${{ matrix.subarch }}:${{ matrix.board }}:link_mode=${{ matrix.link_mode }} CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.build-env.outputs.CORE_HASH }}.tar.bz2 ARTIFACT_TAG: ${{ needs.build-env.outputs.CORE_HASH }}-${{ matrix.board }}-${{ matrix.link_mode }} + CI_BUILDS_URL: ${{ needs.build-env.outputs.CI_BUILDS_URL }} if: ${{ !cancelled() && needs.build-env.result == 'success' }} steps: @@ -386,10 +502,12 @@ jobs: - name: Compile tests for ${{ matrix.board }} uses: pillo79/compile-sketches@next with: + # Use ci-builds core, if possible, to use the exact set of tools required by the source fqbn: ${{ env.FQBN }} platforms: | # Use Board Manager version first to install the tools - name: ${{ env.PLAT }} + ${{ env.CI_BUILDS_URL != '' && format('source-url: {0}', env.CI_BUILDS_URL) || '' }} # ... then replace that with the local version - name: ${{ env.PLAT }} source-path: "ArduinoCore-zephyr" @@ -397,6 +515,11 @@ jobs: ${{ env.ALL_TESTS }} libraries: | ${{ env.ALL_LIBRARIES }} + cli-compile-flags: | + - '--build-property' + - 'compiler.c.extra_flags=-Wno-type-limits -Wno-missing-field-initializers' + - '--build-property' + - 'compiler.cpp.extra_flags=-Wno-type-limits -Wno-missing-field-initializers' verbose: 'false' enable-deltas-report: 'false' enable-issues-report: 'true' @@ -437,6 +560,13 @@ jobs: GH_REPO: ${{ github.repository }} steps: + - name: Install matplotlib + if: ${{ github.event_name == 'pull_request' }} + run: | + sudo apt-get remove --purge man-db -y # skips the mandb triggers + sudo apt-get update + sudo apt-get install -y --no-install-recommends python3-matplotlib + - uses: actions/checkout@v6 with: fetch-depth: 0 @@ -485,6 +615,37 @@ jobs: # errors due to race conditions with parallel branch runs should not fail the job extra/ci_calc_size_reports.py ${GITHUB_BASE_SHA} sketches-reports/ || true + # generate the size trends report (per-board charts + tables) from the + # delta-annotated reports above; best-effort, same as the delta computation + extra/ci_calc_size_trends.py --input-dir sketches-reports/ --base-sha ${GITHUB_BASE_SHA} \ + --output-html size-trends-${{ needs.build-env.outputs.CORE_HASH }}.html --output-md size-trends.md || true + + # append the size trends summary to the PR comment and the step summary, + # below the build summary + if [ -s size-trends.md ]; then + cat size-trends.md >> comment-request/comment_body + cat size-trends.md >> $GITHUB_STEP_SUMMARY + fi + + # upload the size trends HTML report; linked from the PR comment below + - name: Archive size trends report + id: upload-size-trends + uses: actions/upload-artifact@v7 + if: ${{ github.event_name == 'pull_request' }} + with: + name: size-trends-report + path: size-trends-${{ needs.build-env.outputs.CORE_HASH }}.html + if-no-files-found: ignore + archive: false + + - name: Add size trends artifact link, if available + if: ${{ github.event_name == 'pull_request' && steps.upload-size-trends.outputs.artifact-id }} + run: | + ARTIFACT_ID=${{ steps.upload-size-trends.outputs.artifact-id }} + ARTIFACT_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/${ARTIFACT_ID}" + echo -e "\n[Full size trends report](${ARTIFACT_URL})" >> comment-request/comment_body + echo -e "\n[Full size trends report](${ARTIFACT_URL})" >> $GITHUB_STEP_SUMMARY + # upload build metadata artifact, for upload to ci-builds - name: Archive build metadata information uses: actions/upload-artifact@v7 @@ -502,15 +663,6 @@ jobs: path: comment-request/ retention-days: 1 - # upload size delta report artifact (will be retrieved by report_size_deltas.yml) - - name: Archive size deltas report information - uses: actions/upload-artifact@v7 - if: ${{ github.event_name == 'pull_request' }} - with: - name: sketches-reports - path: sketches-reports/ - retention-days: 1 - # upload new official test size artifact (for AWS storage) - name: Archive sketch size information uses: actions/upload-artifact@v7 @@ -566,7 +718,7 @@ jobs: publish-core: name: Publish core runs-on: ubuntu-latest - if: ${{ github.event_name == 'push' && github.repository == 'arduino/ArduinoCore-zephyr' }} + if: ${{ github.event_name == 'push' && github.repository == 'arduino/ArduinoCore-zephyr' && !startsWith(github.ref_name, 'dependabot/') }} needs: - build-env - package-core @@ -601,6 +753,8 @@ jobs: done aws s3 cp size-reports-*.tar.bz2 s3://${{ secrets.S3_BUCKET }}/size-reports/ + + # Prepare the package index JSON snippets for the newly built core packages. prepare-json: @@ -650,9 +804,15 @@ jobs: name: snippets-${{ env.CORE_TAG }} path: snippets/ + + + # Upload the newly built core packages and package index snippets to the + # ci-builds repository so the build is fully logged and the next PRs can + # use them as a reference for testing. + ci-builds-upload: name: Upload to ci-builds - if: ${{ github.event_name == 'push' && github.repository == 'arduino/ArduinoCore-zephyr' }} + if: ${{ github.event_name == 'push' && github.repository == 'arduino/ArduinoCore-zephyr' && !startsWith(github.ref_name, 'dependabot/') }} runs-on: ubuntu-latest needs: - build-env diff --git a/.github/workflows/report_size_deltas.yml b/.github/workflows/report_size_deltas.yml deleted file mode 100644 index adc4abf13..000000000 --- a/.github/workflows/report_size_deltas.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) Arduino s.r.l. and/or its affiliated companies -# SPDX-License-Identifier: Apache-2.0 - -name: Report size deltas - -on: - workflow_run: - workflows: ["Package, test and upload core"] - types: - - completed - -permissions: - contents: read - pull-requests: write - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: arduino/report-size-deltas@v1.1.0 diff --git a/.gitignore b/.gitignore index 53a2144bc..8965cf832 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,11 @@ /firmwares/* /venv/ /boards.local.txt +/platform.local.txt +/loader/VERSION +/cores/arduino/api/ + +__pycache__/ llext-edk/ cflags.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 90b51eead..59aeebe90 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 +# Do nothing unless the Arduino API is enabled in the Zephyr project +if(NOT CONFIG_ARDUINO_API) + return() +endif() + +# Keep this in sync with zephyr/module.yml +set(ARDUINO_API_BLOB_VERSION 1.5.2) + string(REPLACE "__" "_" NORMALIZED_BOARD_TARGET "${NORMALIZED_BOARD_TARGET}") # Define custom pinctrl state "arduino" for dynamic pin multiplexing. @@ -12,14 +20,46 @@ if (IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/variants/${BOARD}) elseif (IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/variants/${NORMALIZED_BOARD_TARGET}) set(variant_dir variants/${NORMALIZED_BOARD_TARGET}) else() -if (CONFIG_ARDUINO_API) message(FATAL_ERROR "Variant dir not found: variants/${BOARD}, variants/${NORMALIZED_BOARD_TARGET}") endif() + +# API archives embed an 'ArduinoCore-API-' directory +set(ARDUINO_API_BLOB_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/zephyr/blobs/ArduinoCore-API/) +set(ARDUINO_API_BLOB_DIR ${ARDUINO_API_BLOB_ROOT_DIR}/ArduinoCore-API-${ARDUINO_API_BLOB_VERSION}) +set(ARDUINO_API_BLOB_ARCHIVE ${ARDUINO_API_BLOB_DIR}.tar.gz) + +if(NOT EXISTS ${ARDUINO_API_BLOB_ARCHIVE}) + message(FATAL_ERROR "Arduino API ${ARDUINO_API_BLOB_VERSION} must be downloaded with 'west blobs fetch'.") +endif() + +if(NOT IS_DIRECTORY ${ARDUINO_API_BLOB_DIR}) + # A new version of the API was downloaded, so extract it to the blob + # directory and copy it to cores/arduino/api + file(ARCHIVE_EXTRACT + INPUT ${ARDUINO_API_BLOB_ARCHIVE} + DESTINATION ${ARDUINO_API_BLOB_ROOT_DIR} + PATTERNS */api/* # only extract the 'api' directory from the archive + ) + set(update_api_copy TRUE) +elseif(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/cores/arduino/api) + # There's nothing where the API should be, copy the extracted API files + set(update_api_copy TRUE) +else() + set(update_api_copy FALSE) endif() -if (CONFIG_ARDUINO_API AND NOT CONFIG_LLEXT) -add_subdirectory(cores) -add_subdirectory(libraries) -zephyr_include_directories(${variant_dir}) +if(update_api_copy) + file(REMOVE_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/cores/arduino/api) + file(COPY ${ARDUINO_API_BLOB_DIR}/api DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/cores/arduino) endif() +if (NOT CONFIG_LLEXT_EDK) + add_subdirectory(cores) + add_subdirectory(libraries) + zephyr_include_directories(${variant_dir}) + + # It's fine to use GLOB here because the API files will only change when + # ARDUINO_API_BLOB_VERSION is modified above, triggering a re-run of CMake. + file(GLOB ARDUINO_API_SOURCES "cores/arduino/api/*.cpp") + zephyr_sources(${ARDUINO_API_SOURCES}) +endif() diff --git a/Kconfig b/Kconfig index edd4710db..b6c45a10d 100644 --- a/Kconfig +++ b/Kconfig @@ -12,6 +12,7 @@ config ARDUINO_API imply NEWLIB_LIBC_FLOAT_PRINTF imply CBPRINTF_FP_SUPPORT imply RING_BUFFER + imply PINCTRL_KEEP_SLEEP_STATE select UART_INTERRUPT_DRIVEN default n @@ -24,7 +25,7 @@ config QEMU_ICOUNT config ARDUINO_API_SERIAL_BUFFER_SIZE int "Buffer size for Arduino Serial API" - default 64 + default 1024 config ARDUINO_ENTRY bool "Provide arduino setup and loop entry points" diff --git a/README.gsoc.md b/README.gsoc.md index 2d2309097..b40655fd5 100644 --- a/README.gsoc.md +++ b/README.gsoc.md @@ -26,19 +26,11 @@ The **Arduino Core API** module for zephyr leverages the power of Zephyr under a west update ``` -* **Note:** For ***Linux users only*** there exists an ``install.sh`` script in this project that can be run to quickly link the ArduinoCore-API to this module. -If you are able to run that script successfully then you can skip the next steps. +* You also need to fetch the ArduinoCore-API sources using the following command: -* To "complete" the core you need to copy or symlink the api folder from the [ArduinoCore-API](https://github.com/arduino/ArduinoCore-API.git) repo to the target's ``cores/arduino`` folder: ```sh -$ git clone git@github.com:arduino/ArduinoCore-API # Any location -$ ln -s ///ArduinoCore-API/api cores/arduino/. +west blobs fetch arduino-api ``` -The `cores` folder can be found at `~/zephyrproject/modules/lib/Arduino-Zephyr-API/cores`. - -**Known Bug(s):** - -__NOTE:__ You can skip this step as well if you ran ``install.sh``. **Maintainers**: - [DhruvaG2000](https://github.com/DhruvaG2000) diff --git a/README.md b/README.md index e8ab1b0a5..ddef61b37 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,6 @@ -> [!IMPORTANT] -> This core is in **BETA**. πŸ§ͺ -> Features may change, and bugs may be present. Use for testing only and provide feedback to help us improve. -> -> [![Default branch status](https://github.com/arduino/ArduinoCore-zephyr/actions/workflows/package_core.yml/badge.svg?branch=arduino&event=push)](https://github.com/arduino/ArduinoCore-zephyr/actions/workflows/package_core.yml) +[![Default branch status](https://github.com/arduino/ArduinoCore-zephyr/actions/workflows/package_core.yml/badge.svg?branch=main&event=push)](https://github.com/arduino/ArduinoCore-zephyr/actions/workflows/package_core.yml) -# 🚧 Arduino Core for Zephyr +# πŸͺ Arduino Core for Zephyr This repository is a downstream fork of the [Arduino Core for Zephyr RTOS-based boards](https://github.com/zephyrproject-rtos/arduino-core-zephyr) that @@ -13,7 +9,12 @@ by the [Arduino IDE](https://docs.arduino.cc/software/ide/), [Arduino CLI](https://docs.arduino.cc/arduino-cli/) and [Arduino App Lab](https://docs.arduino.cc/software/app-lab/). -## 🧐 What is Zephyr? +This core is designed to replace the [mbed OS-based Arduino +Core](https://github.com/arduino/ArduinoCore-mbed) on all the devices it +supported, and to provide a more modern and flexible foundation for current and +future Arduino boards by allowing the use of Zephyr RTOS features and APIs. + +## 🧐 What is Zephyr? [Zephyr RTOS](https://zephyrproject.org/) is an open-source, real-time operating system designed for low-power, resource-constrained devices. It's modular, scalable, and supports multiple architectures. @@ -25,25 +26,32 @@ Install the core and its toolchains via Board Manager: * Download and install the latest [Arduino IDE](https://www.arduino.cc/en/software) (only versions `2.x.x` are supported). * Open the *'Settings / Preferences'* window. * Open the *'Boards Manager'* from the side menu and search for *'Zephyr'*. - * If it doesn’t appear, add the following URL to the *'Additional Boards Manager URLs'* field: `https://downloads.arduino.cc/packages/package_zephyr_index.json` (if you have multiple URLs, separate them with a comma). -* Install the `Arduino Zephyr Boards` platform. +* Install the *'Arduino Zephyr Boards'* platform (or the *'Arduino Uno Q Board'* platform if you have an Arduino UNO Q). -Alternatively, to install the core using the command line, run the following command with the Arduino CLI: +## πŸ—οΈ First Use -```bash -arduino-cli core install arduino:zephyr --additional-urls https://downloads.arduino.cc/packages/package_zephyr_index.json -``` +The first time you use a Zephyr board, the *Zephyr loader* must be installed on +the board. Since release 0.90.0 this procedure is fully automated and the +loader will be installed automatically when you upload your first sketch. -## πŸ—οΈ First Use +[!NOTE] +> The Arduino Portenta C33 needs a full bootloader update to work with the +> Zephyr core. For this board, the update *must be performed manually* the +> first time. Make sure to follow the instructions in [this Help Center +> article](https://support.arduino.cc/hc/en-us/articles/29180434600476-Install-the-Zephyr-loader-on-your-board) +> to properly update the bootloader on your Portenta C33. + +To manually force a loader update, follow these steps: -To get started with your board: * Put the board in bootloader mode by double-clicking the RESET button. * Run the `Burn Bootloader` option from the IDE/CLI. * Note that due to limitations in the Arduino IDE, you may need to select any programmer from the `Programmers` menu. * Once the bootloader is installed, you can load your first sketch by placing the board into bootloader mode again. -> [!NOTE] -> After the initial setup, future sketches will be loaded automatically without needing to reset the board. +> [!NOTE] +> After the initial setup, future sketches will be loaded automatically without +> needing to reset the board. Each sketch update will also reflash the loader +> if a different version is detected. ## πŸ”§ Troubleshooting @@ -65,7 +73,7 @@ To get started with your board: --- #### **Q: I get an OS crash, like ` os: ***** USAGE FAULT *****`** -**A:** This is usually due to a buffer overflow or coding error in the user's own code. However, since the project is still in beta πŸ§ͺ, a [good bug report](#-bug-reporting) could help identify any issues in our code. +**A:** This is usually due to a buffer overflow or coding error in the user's own code. However, a [good bug report](#-bug-reporting) could help identify any issues in the loader or Zephyr itself. --- @@ -83,6 +91,23 @@ See the [Using the Core in Arduino IDE/CLI](#using-the-core-in-arduino-idecli) s #### **Q: Wi-Fi is not working, or I get `Communication with WiFi module failed!` in the Serial Monitor** **A:** You are probably missing the Wi-Fi firmware, or the firmware is corrupted. Boards should come with the Wi-Fi firmware already flashed, but in case Wi-Fi is not working run the [`FlashFormat`](libraries/Storage/examples/FlashFormat/FlashFormat.ino) sketch to restore the firmware. +--- + +#### **Q: I get a `No binaries found for .` when compiling a custom core** + +**A:** To use the Zephyr core from sources, it is not enough to install them in +your `hardware` folder. You also need to obtain the *Zephyr loader* for the +boards you want to use. Read on from [Under the hood](#-under-the-hood) below +to learn how to build (or get!) those binaries. + +--- + +#### **Q: My sketch running on Portenta C33 takes a long time to start, and the initial serial output is lost** + +**A:** This is caused by legacy firmware running on the Portenta C33's ESP32 Wi-Fi chip. Follow the [instructions](libraries/WiFi/examples/SerialBridge/README.md) to update the firmware to the latest version. +If you want to rollback to the Renesas core, remember to downgrade the firmware to the legacy version. + + ## πŸ“š Libraries A number of libraries are bundled with the core, and many more external @@ -108,10 +133,10 @@ The behavior of the `loader` can be adjusted through the `Mode` menu of the IDE: The most important components of this project are: * [Zephyr based loader](/loader) -* [LLEXT](https://docs.zephyrproject.org/latest/services/llext/index.html) -* [Actual core](/cores/arduino) with [variants](/variants) and the usual [platform](/platform.txt) and [boards](/boards) files -* [ArduinoCore-API](https://github.com/arduino/ArduinoCore-API) -* [zephyr-sketch-tool](/extra/zephyr-sketch-tool) +* [Actual core](/cores/arduino) with [variants](/variants) and the usual [platform](/platform.txt) and [boards](/boards) TXT files +* The official [ArduinoCore-API](https://github.com/arduino/ArduinoCore-API) +* [LLEXT](https://docs.zephyrproject.org/latest/services/llext/index.html) as the glue layer between sketch and loader +* [zephyr-sketch-tool](/extra/zephyr-sketch-tool) for packaging the sketch binary ## πŸƒ Shortcut: using the Core in Arduino IDE/CLI without installing Zephyr @@ -119,28 +144,21 @@ The most important components of this project are: > > If you are only interested in developing features in the [core](/cores/arduino) > or [libraries](/libraries), and do not want to set up a full Zephyr build -> environment, you can use the [`sync-zephyr-artifacts`](/extra/sync-zephyr-artifacts) +> environment, you can use the [`sync-zephyr-artifacts`](/extra/sync-zephyr-artifacts) > utility to download a pre-built version of the files needed to compile > sketches and flash the loader. > > To do so, after cloning this repo, compile the `sync-zephyr-artifacts` -> utility via `go build` and run it as `sync-zephyr-artifacts .` to retrieve -> the precompiled files for the current revision of the core. +> utility via `go build` and run it as `sync-zephyr-artifacts ` +> to retrieve the precompiled files for the current revision of the core. > > Next, follow the instructions in [Using the Core in Arduino IDE/CLI](#using-the-core-in-arduino-idecli) -> or [Using the Core in the Arduino App Lab](#using-the-core-in-the-arduino-app-lab) +> or [Using the Core in the Arduino App Lab](#using-the-core-in-the-arduino-app-lab) > to start using the core in your preferred Arduino software. > Remember to [update the loader on your board](#flash-the-loader) as well. ## πŸ› οΈ Setup a Zephyr build environment -> [!WARNING] -> If you checked out this repo before 0.3.2 was released, please note that -> development has switched to the `main` branch; the old `arduino` branch will -> be removed in the short future. Please follow this -> [migration guide](https://github.com/arduino/ArduinoCore-zephyr/issues/163) -> to update your local branches and outstanding PRs. - In this section, we’ll guide you through setting up your environment to work on and update the Zephyr core. Shell scripts are available to simplify the installation process (Windows is not directly supported at the moment πŸ˜”, but there are some tricks - see below). @@ -172,17 +190,9 @@ There are two strategies to set up the sources for building the loader on Window 2) Install the sources within the WSL file system, like: `~/git/ArduinoCore-zephyr` There are pros and cons to both strategies: -1) Builds on the native Windows file system are relatively very slow, but once done, you can use the results directly within the Arduino IDE. +1) Builds on the native Windows file system are relatively very slow, but once done, you can use the results directly within the Arduino IDE. 2) Builds on WSL's file system are a lot faster, however, you need to copy the resulting build back to somewhere in your Windows directory structure. Use this location in the Arduino IDE as mentioned below in the [Using the Core in Arduino IDE/CLI](#using-the-core-in-arduino-idecli) section. -After `bootstrap.sh` has completed, you may also have to update the `cores\arduino\api` link to the path of the ArduinoCore-API's `api` folder. - -### Clone the repository -```bash -mkdir my_new_zephyr_folder && cd my_new_zephyr_folder -git clone https://github.com/arduino/ArduinoCore-zephyr -``` - ### Run the ```bootstrap``` script ```bash cd ArduinoCore-zephyr @@ -267,12 +277,12 @@ Once this is done, your development folder will appear in the IDE/CLI package list as `arduino-git:zephyr`, and the Fully Qualified Board Name (FQBN) to use will be `arduino-git:zephyr:name_from_boards_txt`. -Remember to also install and/or update the officially published core in the IDE Board Manager to get the latest tools and dependencies. +Remember to also install and/or update the officially published core in the IDE Board Manager to get the latest tools and dependencies. [βš™οΈ Installation](#️-installation). ### Using the Core in the Arduino App Lab -> [!WARNING] +> [!WARNING] > Arduino App Lab expects a hardcoded FQBN (`arduino:zephyr:unoq` for the UNO Q), so the [technique used for Arduino IDE/CLI](#using-the-core-in-arduino-idecli) **does not** work. A small workaround is required. 1. Disable the release core @@ -317,7 +327,7 @@ mv ~/Arduino/hardware/arduino/zephyr ~/Arduino/hardware/arduino/zephyr.disable > ```bash > . venv/bin/activate > west config -d manifest.project-filter -> west sdk install --version 0.17.0 +> west sdk install > west update > ``` @@ -349,18 +359,14 @@ To report a bug, open the [issues](/../../issues) and follow the instructions. A Contributions are always welcome. The preferred way to receive code contribution is by submitting a [Pull request](/../../pulls). -> [!WARNING] -> At this stage of development, we only accept Pull requests for bug fixes and features. We do **not** accept support for new targets. - ## πŸ“Œ Upcoming features - [ ] USB: switch to `USB_DEVICE_STACK_NEXT` to support PluggableUSB -- [x] Relocate RODATA in flash to accommodate sketches with large assets - [ ] Provide better error reporting for failed llext operations - [ ] Replace [`llext_exports.c`](/loader/llext_exports.c) with proper symbols generation (via includes) -- [ ] Fix corner cases with `std::` includes (like ``) - [ ] Get rid of all warnings ## 🌟 Acknowledgments -This effort would have been very hard without the [GSoC project](/README.gsoc.md) and the Zephyr community. +This effort would have been very hard without the [GSoC project](/README.gsoc.md), the Zephyr community and all the contributors to this repo. +Thanks to everyone who has contributed to this project! diff --git a/boards.txt b/boards.txt index e7b6325a1..798564387 100644 --- a/boards.txt +++ b/boards.txt @@ -25,7 +25,7 @@ menu.wait_linux_boot=Startup mode giga.name=Arduino Giga R1 giga.build.core=arduino giga.build.crossprefix=arm-zephyr-eabi- -giga.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ +giga.build.compiler_path={runtime.tools.arm-zephyr-eabi-1.0.1.path}/bin/ giga.menu.debug.false=Standard giga.menu.debug.true=Debug @@ -39,23 +39,20 @@ giga.menu.link_mode.static.upload.extension=bin-zsk.bin giga.build.zephyr_target=arduino_giga_r1//m7 giga.build.zephyr_args=--shield arduino_giga_display_shield giga.build.zephyr_hals=hal_stm32 hal_infineon +giga.build.zephyr_toolchain=arm-zephyr-eabi giga.build.variant=arduino_giga_r1_stm32h747xx_m7 giga.build.artifact=zephyr_main giga.build.extra_flags= giga.build.postbuild.cmd="{tools.imgtool.path}/{tools.imgtool.cmd}" exit giga.build.board=GIGA -giga.vid.0=0x2341 -giga.pid.0=0x0066 giga.upload_port.0.vid=0x2341 -giga.upload_port.0.pid=0x0066 +giga.upload_port.0.pid=0x0666 giga.upload.tool=dfu-util giga.upload.tool.default=dfu-util giga.upload.protocol= giga.upload.transport= -giga.upload.vid=0x2341 -giga.upload.pid=0x0366 giga.upload.interface=0 giga.upload.use_1200bps_touch=true giga.upload.wait_for_upload_port=true @@ -63,7 +60,6 @@ giga.upload.native_usb=true giga.bootloader.tool=dfu-util giga.bootloader.tool.default=dfu-util -giga.bootloader.vid=0x2341 giga.bootloader.pid=0x0366 giga.bootloader.interface=0 giga.bootloader.file=zephyr-{build.variant}.bin @@ -81,7 +77,7 @@ giga.debug.svd_file={runtime.platform.path}/svd/STM32H747_CM7.svd nano33ble.name=Arduino Nano 33 BLE nano33ble.build.core=arduino nano33ble.build.crossprefix=arm-zephyr-eabi- -nano33ble.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ +nano33ble.build.compiler_path={runtime.tools.arm-zephyr-eabi-1.0.1.path}/bin/ nano33ble.menu.debug.false=Standard nano33ble.menu.debug.true=Debug @@ -95,24 +91,20 @@ nano33ble.menu.link_mode.static.upload.extension=bin-zsk.bin nano33ble.build.zephyr_target=arduino_nano_33_ble//sense nano33ble.build.zephyr_args= nano33ble.build.zephyr_hals=hal_nordic +nano33ble.build.zephyr_toolchain=arm-zephyr-eabi nano33ble.build.artifact=zephyr_main nano33ble.build.variant=arduino_nano_33_ble_nrf52840_sense nano33ble.build.extra_flags= nano33ble.build.postbuild.cmd="{tools.imgtool.path}/{tools.imgtool.cmd}" exit nano33ble.build.board=NANO33BLE -nano33ble.vid.0=0x2341 -nano33ble.pid.0=0x035a nano33ble.upload_port.0.vid=0x2341 -nano33ble.upload_port.0.pid=0x005a +nano33ble.upload_port.0.pid=0x065a nano33ble.upload.tool=bossac nano33ble.upload.tool.default=bossac nano33ble.upload.protocol= nano33ble.upload.transport= -nano33ble.upload.vid=0x2341 -nano33ble.upload.pid=0x005a -nano33ble.upload.interface=0 nano33ble.upload.use_1200bps_touch=true nano33ble.upload.wait_for_upload_port=true nano33ble.upload.native_usb=true @@ -120,9 +112,6 @@ nano33ble.upload.offset=0x10000 # bootloader skips its own partition nano33ble.bootloader.tool=bossac nano33ble.bootloader.tool.default=bossac -nano33ble.bootloader.vid=0x2341 -nano33ble.bootloader.pid=0x035b -nano33ble.bootloader.interface=0 nano33ble.bootloader.file=zephyr-{build.variant}.bin nano33ble.bootloader.address=0x0000 @@ -137,7 +126,7 @@ nano33ble.debug.cortex-debug.custom.request=attach ek_ra8d1.name=Renesas RA8D1 EK ek_ra8d1.build.core=arduino ek_ra8d1.build.crossprefix=arm-zephyr-eabi- -ek_ra8d1.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ +ek_ra8d1.build.compiler_path={runtime.tools.arm-zephyr-eabi-1.0.1.path}/bin/ ek_ra8d1.menu.debug.false=Standard ek_ra8d1.menu.debug.true=Debug @@ -151,6 +140,7 @@ ek_ra8d1.menu.link_mode.static.upload.extension=bin-zsk.bin ek_ra8d1.build.zephyr_target=ek_ra8d1 ek_ra8d1.build.zephyr_args= ek_ra8d1.build.zephyr_hals=hal_renesas +ek_ra8d1.build.zephyr_toolchain=arm-zephyr-eabi ek_ra8d1.build.variant=ek_ra8d1_r7fa8d1bhecbd ek_ra8d1.build.extra_flags= @@ -160,18 +150,13 @@ ek_ra8d1.build.board=EK_RA8D1 #ek_ra8d1.recipe.hooks.objcopy.postobjcopy.4.pattern=cp {build.variant.path}/flasher.jlink "{build.path}/flasher.jlink" #ek_ra8d1.recipe.hooks.objcopy.postobjcopy.5.pattern=sed -i 's|SKETCH|"{build.path}/{build.project_name}.llext.dfu.bin"|g' "{build.path}/flasher.jlink" -ek_ra8d1.vid.0=0x2341 -ek_ra8d1.pid.0=0x0077 -ek_ra8d1.upload_port.0.vid=0x2341 -ek_ra8d1.upload_port.0.pid=0x0077 +ek_ra8d1.upload_port.0.vid=0x045b +ek_ra8d1.upload_port.0.pid=0x0261 ek_ra8d1.upload.tool=pyocd ek_ra8d1.upload.tool.default=pyocd ek_ra8d1.upload.protocol= ek_ra8d1.upload.transport= -ek_ra8d1.upload.vid=0x2341 -ek_ra8d1.upload.pid=0x0366 -ek_ra8d1.upload.interface=0 ek_ra8d1.upload.use_1200bps_touch=false ek_ra8d1.upload.wait_for_upload_port=false ek_ra8d1.upload.native_usb=true @@ -188,7 +173,7 @@ ek_ra8d1.bootloader.target=R7FA8D1AH frdm_mcxn947.name=NXP FRDM MCXN947 frdm_mcxn947.build.core=arduino frdm_mcxn947.build.crossprefix=arm-zephyr-eabi- -frdm_mcxn947.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ +frdm_mcxn947.build.compiler_path={runtime.tools.arm-zephyr-eabi-1.0.1.path}/bin/ frdm_mcxn947.menu.debug.false=Standard frdm_mcxn947.menu.debug.true=Debug @@ -202,13 +187,12 @@ frdm_mcxn947.menu.link_mode.static.upload.extension=bin-zsk.bin frdm_mcxn947.build.zephyr_target=frdm_mcxn947//cpu0 frdm_mcxn947.build.zephyr_args= frdm_mcxn947.build.zephyr_hals=hal_nxp +frdm_mcxn947.build.zephyr_toolchain=arm-zephyr-eabi frdm_mcxn947.build.variant=frdm_mcxn947_mcxn947_cpu0 frdm_mcxn947.build.extra_flags= frdm_mcxn947.build.postbuild.cmd="{tools.imgtool.path}/{tools.imgtool.cmd}" exit frdm_mcxn947.build.board=FRDM_MCXN947 -frdm_mcxn947.vid.0=0x1fc9 -frdm_mcxn947.pid.0=0x0143 frdm_mcxn947.upload_port.0.vid=0x1fc9 frdm_mcxn947.upload_port.0.pid=0x0143 frdm_mcxn947.upload.target=mcxn947vdf @@ -217,9 +201,6 @@ frdm_mcxn947.upload.tool=pyocd frdm_mcxn947.upload.tool.default=pyocd frdm_mcxn947.upload.protocol= frdm_mcxn947.upload.transport= -frdm_mcxn947.upload.vid=0x1fc9 -frdm_mcxn947.upload.pid=0x0143 -frdm_mcxn947.upload.interface=0 frdm_mcxn947.upload.use_1200bps_touch=false frdm_mcxn947.upload.wait_for_upload_port=false frdm_mcxn947.upload.native_usb=true @@ -235,7 +216,7 @@ frdm_mcxn947.bootloader.target=mcxn947vdf portentah7.name=Arduino Portenta H7 portentah7.build.core=arduino portentah7.build.crossprefix=arm-zephyr-eabi- -portentah7.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ +portentah7.build.compiler_path={runtime.tools.arm-zephyr-eabi-1.0.1.path}/bin/ portentah7.menu.debug.false=Standard portentah7.menu.debug.true=Debug @@ -249,23 +230,20 @@ portentah7.menu.link_mode.static.upload.extension=bin-zsk.bin portentah7.build.zephyr_target=arduino_portenta_h7@1.0.0//m7 portentah7.build.zephyr_args= portentah7.build.zephyr_hals=hal_stm32 hal_infineon +portentah7.build.zephyr_toolchain=arm-zephyr-eabi portentah7.build.artifact=zephyr_main portentah7.build.variant=arduino_portenta_h7_stm32h747xx_m7 portentah7.build.extra_flags= portentah7.build.postbuild.cmd="{tools.imgtool.path}/{tools.imgtool.cmd}" exit portentah7.build.board=PORTENTA_H7_M7 -portentah7.vid.0=0x2341 -portentah7.pid.0=0x005b portentah7.upload_port.0.vid=0x2341 -portentah7.upload_port.0.pid=0x035b +portentah7.upload_port.0.pid=0x065b portentah7.upload.tool=dfu-util portentah7.upload.tool.default=dfu-util portentah7.upload.protocol= portentah7.upload.transport= -portentah7.upload.vid=0x2341 -portentah7.upload.pid=0x035b portentah7.upload.interface=0 portentah7.upload.use_1200bps_touch=true portentah7.upload.wait_for_upload_port=true @@ -273,7 +251,6 @@ portentah7.upload.native_usb=true portentah7.bootloader.tool=dfu-util portentah7.bootloader.tool.default=dfu-util -portentah7.bootloader.vid=0x2341 portentah7.bootloader.pid=0x035b portentah7.bootloader.interface=0 portentah7.bootloader.file=zephyr-{build.variant}.bin @@ -291,7 +268,7 @@ portentah7.debug.svd_file={runtime.platform.path}/svd/STM32H747_CM7.svd nicla_vision.name=Arduino Nicla Vision nicla_vision.build.core=arduino nicla_vision.build.crossprefix=arm-zephyr-eabi- -nicla_vision.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ +nicla_vision.build.compiler_path={runtime.tools.arm-zephyr-eabi-1.0.1.path}/bin/ nicla_vision.menu.debug.false=Standard nicla_vision.menu.debug.true=Debug @@ -305,23 +282,20 @@ nicla_vision.menu.link_mode.static.upload.extension=bin-zsk.bin nicla_vision.build.zephyr_target=arduino_nicla_vision//m7 nicla_vision.build.zephyr_args= nicla_vision.build.zephyr_hals=hal_stm32 hal_infineon +nicla_vision.build.zephyr_toolchain=arm-zephyr-eabi nicla_vision.build.artifact=zephyr_main nicla_vision.build.variant=arduino_nicla_vision_stm32h747xx_m7 nicla_vision.build.extra_flags= nicla_vision.build.postbuild.cmd="{tools.imgtool.path}/{tools.imgtool.cmd}" exit nicla_vision.build.board=NICLA_VISION -nicla_vision.vid.0=0x2341 -nicla_vision.pid.0=0x005b nicla_vision.upload_port.0.vid=0x2341 -nicla_vision.upload_port.0.pid=0x025f +nicla_vision.upload_port.0.pid=0x065f nicla_vision.upload.tool=dfu-util nicla_vision.upload.tool.default=dfu-util nicla_vision.upload.protocol= nicla_vision.upload.transport= -nicla_vision.upload.vid=0x2341 -nicla_vision.upload.pid=0x035f nicla_vision.upload.interface=0 nicla_vision.upload.use_1200bps_touch=true nicla_vision.upload.wait_for_upload_port=true @@ -329,7 +303,6 @@ nicla_vision.upload.native_usb=true nicla_vision.bootloader.tool=dfu-util nicla_vision.bootloader.tool.default=dfu-util -nicla_vision.bootloader.vid=0x2341 nicla_vision.bootloader.pid=0x035f nicla_vision.bootloader.interface=0 nicla_vision.bootloader.file=zephyr-{build.variant}.bin @@ -347,7 +320,7 @@ nicla_vision.debug.svd_file={runtime.platform.path}/svd/STM32H747_CM7.svd frdm_rw612.name=NXP FRDM RW612 frdm_rw612.build.core=arduino frdm_rw612.build.crossprefix=arm-zephyr-eabi- -frdm_rw612.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ +frdm_rw612.build.compiler_path={runtime.tools.arm-zephyr-eabi-1.0.1.path}/bin/ frdm_rw612.menu.debug.false=Standard frdm_rw612.menu.debug.true=Debug @@ -361,13 +334,12 @@ frdm_rw612.menu.link_mode.static.upload.extension=bin-zsk.bin frdm_rw612.build.zephyr_target=frdm_rw612 frdm_rw612.build.zephyr_args= frdm_rw612.build.zephyr_hals=hal_stm32 +frdm_rw612.build.zephyr_toolchain=arm-zephyr-eabi frdm_rw612.build.variant=frdm_rw612_rw612 frdm_rw612.build.extra_flags= frdm_rw612.build.postbuild.cmd="{tools.imgtool.path}/{tools.imgtool.cmd}" exit frdm_rw612.build.board=FRDM_RW612 -frdm_rw612.vid.0=0x1fc9 -frdm_rw612.pid.0=0x0143 frdm_rw612.upload_port.0.vid=0x1fc9 frdm_rw612.upload_port.0.pid=0x0143 frdm_rw612.upload.target=rw612 @@ -376,9 +348,6 @@ frdm_rw612.upload.tool=pyocd frdm_rw612.upload.tool.default=pyocd frdm_rw612.upload.protocol= frdm_rw612.upload.transport= -frdm_rw612.upload.vid=0x1fc9 -frdm_rw612.upload.pid=0x0143 -frdm_rw612.upload.interface=0 frdm_rw612.upload.use_1200bps_touch=false frdm_rw612.upload.wait_for_upload_port=false frdm_rw612.upload.native_usb=true @@ -393,7 +362,7 @@ frdm_rw612.bootloader.target=rw612 nicla_sense.name=Arduino Nicla Sense ME nicla_sense.build.core=arduino nicla_sense.build.crossprefix=arm-zephyr-eabi- -nicla_sense.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ +nicla_sense.build.compiler_path={runtime.tools.arm-zephyr-eabi-1.0.1.path}/bin/ nicla_sense.menu.link_mode.dynamic=Dynamic nicla_sense.menu.link_mode.static=Static @@ -403,6 +372,7 @@ nicla_sense.menu.link_mode.static.upload.extension=bin-zsk.bin nicla_sense.build.zephyr_target=arduino_nicla_sense_me nicla_sense.build.zephyr_args= nicla_sense.build.zephyr_hals=hal_nordic +nicla_sense.build.zephyr_toolchain=arm-zephyr-eabi nicla_sense.build.artifact=zephyr_main nicla_sense.build.variant=arduino_nicla_sense_me_nrf52832 @@ -410,8 +380,6 @@ nicla_sense.openocd_cfg=flash_sketch.cfg nicla_sense.build.extra_flags= nicla_sense.build.postbuild.cmd="{tools.imgtool.path}/{tools.imgtool.cmd}" exit nicla_sense.build.board=NICLA_SENSE_ME -nicla_sense.vid.0=0x2341 -nicla_sense.pid.0=0x0360 nicla_sense.upload_port.0.vid=0x2341 nicla_sense.upload_port.0.pid=0x0060 @@ -420,9 +388,6 @@ nicla_sense.upload.tool.default=openocd nicla_sense.upload.protocol= nicla_sense.upload.config=-f target/nrf52.cfg nicla_sense.upload.programmer=-f interface/cmsis-dap.cfg -nicla_sense.upload.vid=0x2341 -nicla_sense.upload.pid=0x0060 -nicla_sense.upload.interface=0 nicla_sense.upload.use_1200bps_touch=false nicla_sense.upload.wait_for_upload_port=false nicla_sense.upload.native_usb=false @@ -430,9 +395,6 @@ nicla_sense.upload.target=nrf52 nicla_sense.bootloader.tool=openocd nicla_sense.bootloader.tool.default=openocd -nicla_sense.bootloader.vid=0x2341 -nicla_sense.bootloader.pid=0x0360 -nicla_sense.bootloader.interface=0 nicla_sense.bootloader.file=zephyr-{build.variant}.hex nicla_sense.bootloader.target=nrf52 @@ -448,7 +410,7 @@ nicla_sense.debug.cortex-debug.custom.request=attach portentac33.name=Arduino Portenta C33 portentac33.build.core=arduino portentac33.build.crossprefix=arm-zephyr-eabi- -portentac33.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ +portentac33.build.compiler_path={runtime.tools.arm-zephyr-eabi-1.0.1.path}/bin/ portentac33.menu.debug.false=Standard portentac33.menu.debug.true=Debug @@ -462,43 +424,46 @@ portentac33.menu.link_mode.static.upload.extension=bin-zsk.bin portentac33.build.zephyr_target=arduino_portenta_c33 portentac33.build.zephyr_args= portentac33.build.zephyr_hals=hal_renesas nanopb +portentac33.build.zephyr_toolchain=arm-zephyr-eabi portentac33.build.artifact=zephyr_main portentac33.build.variant=arduino_portenta_c33_r7fa6m5bh3cfc portentac33.build.extra_flags= portentac33.build.postbuild.cmd="{tools.imgtool.path}/{tools.imgtool.cmd}" exit portentac33.build.board=PORTENTA_C33 -portentac33.vid.0=0x2341 -portentac33.pid.0=0x0068 portentac33.upload_port.0.vid=0x2341 -portentac33.upload_port.0.pid=0x0068 +portentac33.upload_port.0.pid=0x0668 portentac33.upload.target=portentac33 portentac33.upload.tool=dfu-util portentac33.upload.tool.default=dfu-util portentac33.upload.protocol= portentac33.upload.transport= -portentac33.upload.vid=0x2341 -portentac33.upload.pid=0x0368 portentac33.upload.interface=1 portentac33.upload.use_1200bps_touch=true portentac33.upload.wait_for_upload_port=false portentac33.upload.native_usb=true -portentac33.upload.dfuse=-Q -w +portentac33.upload.dfuse= portentac33.bootloader.tool=dfu-util portentac33.bootloader.tool.default=dfu-util portentac33.bootloader.file=zephyr-{build.variant}.bin portentac33.bootloader.interface=0 portentac33.bootloader.address=0x10000 -portentac33.bootloader.dfuse=-Q +portentac33.bootloader.pid=0x0368 +portentac33.bootloader.dfuse= + +# the space before the extra arguments is important +portentac33.leave.dfuse={empty} -Q +portentac33.loader.args.with_sketch={empty} -w +portentac33.sketch.args.without_loader={empty} -w ########################################################################################## opta.name=Arduino Opta opta.build.core=arduino opta.build.crossprefix=arm-zephyr-eabi- -opta.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ +opta.build.compiler_path={runtime.tools.arm-zephyr-eabi-1.0.1.path}/bin/ opta.menu.debug.false=Standard opta.menu.debug.true=Debug @@ -512,23 +477,20 @@ opta.menu.link_mode.static.upload.extension=bin-zsk.bin opta.build.zephyr_target=arduino_opta//m7 opta.build.zephyr_args= opta.build.zephyr_hals=hal_stm32 hal_infineon +opta.build.zephyr_toolchain=arm-zephyr-eabi opta.build.artifact=zephyr_main opta.build.variant=arduino_opta_stm32h747xx_m7 opta.build.extra_flags= opta.build.postbuild.cmd="{tools.imgtool.path}/{tools.imgtool.cmd}" exit opta.build.board=OPTA -opta.vid.0=0x2341 -opta.pid.0=0x0064 opta.upload_port.0.vid=0x2341 -opta.upload_port.0.pid=0x0364 +opta.upload_port.0.pid=0x0664 opta.upload.tool=dfu-util opta.upload.tool.default=dfu-util opta.upload.protocol= opta.upload.transport= -opta.upload.vid=0x2341 -opta.upload.pid=0x0364 opta.upload.interface=0 opta.upload.use_1200bps_touch=true opta.upload.wait_for_upload_port=true @@ -536,7 +498,6 @@ opta.upload.native_usb=true opta.bootloader.tool=dfu-util opta.bootloader.tool.default=dfu-util -opta.bootloader.vid=0x2341 opta.bootloader.pid=0x0364 opta.bootloader.interface=0 opta.bootloader.file=zephyr-{build.variant}.bin @@ -554,7 +515,7 @@ opta.debug.svd_file={runtime.platform.path}/svd/STM32H747_CM7.svd nano_matter.name=Arduino Nano Matter nano_matter.build.core=arduino nano_matter.build.crossprefix=arm-zephyr-eabi- -nano_matter.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ +nano_matter.build.compiler_path={runtime.tools.arm-zephyr-eabi-1.0.1.path}/bin/ nano_matter.menu.link_mode.dynamic=Dynamic nano_matter.menu.link_mode.static=Static @@ -564,6 +525,7 @@ nano_matter.menu.link_mode.static.upload.extension=bin-zsk.bin nano_matter.build.zephyr_target=arduino_nano_matter nano_matter.build.zephyr_args= nano_matter.build.zephyr_hals=hal_silabs +nano_matter.build.zephyr_toolchain=arm-zephyr-eabi nano_matter.build.artifact=zephyr_main nano_matter.build.variant=arduino_nano_matter_mgm240sd22vna @@ -571,8 +533,6 @@ nano_matter.openocd_cfg=flash_sketch.cfg nano_matter.build.extra_flags= nano_matter.build.postbuild.cmd="{tools.imgtool.path}/{tools.imgtool.cmd}" exit nano_matter.build.board=ARDUINO_NANO_MATTER -nano_matter.vid.0=0x2341 -nano_matter.pid.0=0x0072 nano_matter.upload_port.0.vid=0x2341 nano_matter.upload_port.0.pid=0x0072 @@ -588,9 +548,6 @@ nano_matter.upload.native_usb=false nano_matter.bootloader.tool=openocd nano_matter.bootloader.tool.default=openocd -nano_matter.bootloader.vid=0x2341 -nano_matter.bootloader.pid=0x0072 -nano_matter.bootloader.interface=0 nano_matter.bootloader.file=zephyr-{build.variant}.hex nano_matter.bootloader.target=arduino_nano_matter @@ -612,7 +569,7 @@ nano_matter.debug.cortex-debug.custom.request=attach unoq.name=Arduino UNO Q unoq.build.core=arduino unoq.build.crossprefix=arm-zephyr-eabi- -unoq.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ +unoq.build.compiler_path={runtime.tools.arm-zephyr-eabi-1.0.1.path}/bin/ unoq.menu.link_mode.dynamic=Dynamic unoq.menu.link_mode.static=Static @@ -627,6 +584,8 @@ unoq.menu.wait_linux_boot.app.build.boot_mode=app unoq.build.zephyr_target=arduino_uno_q unoq.build.zephyr_args= +unoq.build.zephyr_hals=hal_stm32 +unoq.build.zephyr_toolchain=arm-zephyr-eabi unoq.build.variant=arduino_uno_q_stm32u585xx unoq.build.artifact=zephyr_unoq unoq.build.subarch=zephyr @@ -635,10 +594,6 @@ unoq.openocd_cfg=flash_sketch.cfg unoq.build.extra_flags= unoq.build.postbuild.cmd="{tools.imgtool.path}/{tools.imgtool.cmd}" exit unoq.build.board=UNO_Q -unoq.build.zephyr_hals=hal_stm32 -unoq.build.zephyr_toolchain=arm-zephyr-eabi -unoq.vid.0=0x2341 -unoq.pid.0=0x0078 unoq.upload_port.0.vid=0x2341 unoq.upload_port.0.pid=0x0078 unoq.upload.target=stm32u585zitxq @@ -648,9 +603,6 @@ unoq.upload.tool.default=remoteocd unoq.upload.tool.network=remoteocd_network unoq.upload.protocol= unoq.upload.transport= -unoq.upload.vid=0x2341 -unoq.upload.pid=0x0078 -unoq.upload.interface=0 unoq.upload.use_1200bps_touch=false unoq.upload.wait_for_upload_port=false unoq.upload.native_usb=true @@ -665,7 +617,7 @@ unoq.bootloader.target=stm32u585zitxq nano_connect.name=Arduino Nano RP2040 Connect nano_connect.build.core=arduino nano_connect.build.crossprefix=arm-zephyr-eabi- -nano_connect.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/ +nano_connect.build.compiler_path={runtime.tools.arm-zephyr-eabi-1.0.1.path}/bin/ nano_connect.menu.debug.false=Standard nano_connect.menu.debug.true=Debug @@ -679,46 +631,31 @@ nano_connect.menu.link_mode.static.upload.extension=bin-zsk.bin nano_connect.build.zephyr_target=arduino_nano_connect nano_connect.build.zephyr_args= nano_connect.build.zephyr_hals=hal_rpi_pico +nano_connect.build.zephyr_toolchain=arm-zephyr-eabi nano_connect.build.artifact=zephyr_main nano_connect.build.variant=arduino_nano_connect_rp2040 -nano_connect.build.mcu=cortex-m0plus -nano_connect.build.fpu= -nano_connect.build.architecture=cortex-m0plus -nano_connect.compiler.zephyr.arch.define= -nano_connect.build.float-abi=-mfloat-abi=soft nano_connect.build.extra_flags= nano_connect.build.postbuild.cmd="{tools.imgtool.path}/{tools.imgtool.cmd}" exit -nano_connect.recipe.hooks.objcopy.postobjcopy.3.pattern="{runtime.tools.bin2uf2.path}/bin2uf2" {upload.address} {upload.familyid} "{build.path}/{build.project_name}.{upload.extension}" "{build.path}/{build.project_name}.uf2" +nano_connect.recipe.hooks.objcopy.postobjcopy.3.pattern="{runtime.tools.bin2uf2.path}/bin2uf2" {upload.address} {upload.familyid} "{build.path}/{build.project_name}.{upload.extension}" "{build.path}/{build.project_name}.{upload.extension}.uf2" +nano_connect.upload.artifacts.sketch={build.path}/{build.project_name}.{upload.extension}.uf2 nano_connect.build.board=NANO_RP2040_CONNECT -nano_connect.compiler.zephyr= -nano_connect.vid.0=0x2341 -nano_connect.pid.0=0x005E nano_connect.upload_port.0.vid=0x2341 -nano_connect.upload_port.0.pid=0x005E +nano_connect.upload_port.0.pid=0x065E nano_connect.upload.tool=picotool nano_connect.upload.tool.default=picotool nano_connect.upload.protocol= nano_connect.upload.transport= -nano_connect.upload.vid=0x2341 -nano_connect.upload.pid=0x005E -nano_connect.upload.interface=0 nano_connect.upload.use_1200bps_touch=true nano_connect.upload.wait_for_upload_port=false nano_connect.upload.native_usb=true -nano_connect.upload.maximum_size=16777216 -nano_connect.upload.maximum_data_size=270336 -nano_connect.upload.address=0x100E0000 nano_connect.upload.familyid=0xe48bff56 nano_connect.bootloader.tool=picotool nano_connect.bootloader.tool.default=picotool -nano_connect.bootloader.vid=0x2341 -nano_connect.bootloader.pid=0x005E -nano_connect.bootloader.interface=0 -nano_connect.bootloader.file=zephyr-{build.variant} +nano_connect.bootloader.file=zephyr-{build.variant}.uf2 nano_connect.debug.tool=gdb nano_connect.debug.server.openocd.scripts.0=interface/{programmer.protocol}.cfg @@ -728,3 +665,65 @@ nano_connect.debug.cortex-debug.custom.request=attach nano_connect.debug.svd_file={runtime.platform.path}/svd/rp2040.svd ############################################################################################################## + +ventunoq.name=Arduino Ventuno Q +ventunoq.build.core=arduino +ventunoq.build.crossprefix=arm-zephyr-eabi- +ventunoq.build.compiler_path={runtime.tools.arm-zephyr-eabi-1.0.1.path}/bin/ + +ventunoq.menu.link_mode.dynamic=Dynamic +ventunoq.menu.link_mode.static=Static +ventunoq.menu.link_mode.static.build.link_mode=static +ventunoq.menu.link_mode.static.upload.extension=bin-zsk.bin + +ventunoq.menu.wait_linux_boot.yes=Wait for Linux +ventunoq.menu.wait_linux_boot.no=Immediate +ventunoq.menu.wait_linux_boot.no.build.boot_mode=immediate +ventunoq.menu.wait_linux_boot.app=Wait for App +ventunoq.menu.wait_linux_boot.app.build.boot_mode=app +ventunoq.openocd_cfg=flash_sketch.cfg + +ventunoq.build.zephyr_target=arduino_ventuno_q +ventunoq.build.zephyr_args= +ventunoq.build.variant=arduino_ventuno_q_stm32h5f5xx +ventunoq.build.artifact=zephyr_unoq +ventunoq.build.subarch=zephyr +ventunoq.build.mcu=cortex-m33 +ventunoq.build.fpu=-mfpu=fpv5-sp-d16 +ventunoq.build.architecture=cortex-m33 +ventunoq.compiler.zephyr.arch.define= + +ventunoq.build.float-abi=-mfloat-abi=hard +ventunoq.build.extra_flags= +ventunoq.build.postbuild.cmd="{tools.imgtool.path}/{tools.imgtool.cmd}" exit +ventunoq.build.board=VENTUNO_Q +ventunoq.build.zephyr_hals=hal_stm32 +ventunoq.build.extra_flags= +ventunoq.compiler.zephyr= +ventunoq.vid.0=0x2341 +ventunoq.pid.0=0x007a +ventunoq.vid.1=0x1209 +ventunoq.pid.1=0xca01 +ventunoq.upload_port.0.vid=0x2341 +ventunoq.upload_port.0.pid=0x007a +ventunoq.upload.address=0x80F0000 +ventunoq.upload.target=stm32h5f5xx + +ventunoq.upload.tool=remoteocd +ventunoq.upload.tool.default=remoteocd +ventunoq.upload.tool.network=remoteocd_network +ventunoq.upload.protocol= +ventunoq.upload.transport= +ventunoq.upload.vid=0x2341 +ventunoq.upload.pid=0x007a +ventunoq.upload.interface=0 +ventunoq.upload.use_1200bps_touch=false +ventunoq.upload.wait_for_upload_port=false +ventunoq.upload.native_usb=true +ventunoq.upload.maximum_size=1966080 +ventunoq.upload.maximum_data_size=523624 + +ventunoq.bootloader.tool=remoteocd +ventunoq.bootloader.tool.default=remoteocd +ventunoq.bootloader.file=zephyr-{build.variant}.elf +ventunoq.bootloader.target=stm32h5f5xx diff --git a/cores/CMakeLists.txt b/cores/CMakeLists.txt index 3ce59d21d..626fe6b12 100644 --- a/cores/CMakeLists.txt +++ b/cores/CMakeLists.txt @@ -1,15 +1,3 @@ # SPDX-License-Identifier: Apache-2.0 add_subdirectory(arduino) - -zephyr_include_directories(../../ArduinoCore-API/) - -if(NOT DEFINED ARDUINO_BUILD_PATH) -zephyr_sources(../../ArduinoCore-API/api/CanMsg.cpp) -zephyr_sources(../../ArduinoCore-API/api/CanMsgRingbuffer.cpp) -zephyr_sources(../../ArduinoCore-API/api/Common.cpp) -zephyr_sources(../../ArduinoCore-API/api/IPAddress.cpp) -zephyr_sources(../../ArduinoCore-API/api/Print.cpp) -zephyr_sources(../../ArduinoCore-API/api/Stream.cpp) -zephyr_sources(../../ArduinoCore-API/api/String.cpp) -endif() diff --git a/cores/arduino/Arduino.h b/cores/arduino/Arduino.h index 02fd95a61..446171844 100644 --- a/cores/arduino/Arduino.h +++ b/cores/arduino/Arduino.h @@ -8,6 +8,10 @@ #include "api/ArduinoAPI.h" +// redefined by Zephyr below +#undef min +#undef max + #include #include #include diff --git a/cores/arduino/SerialUSB.h b/cores/arduino/SerialUSB.h index 3be3f419f..75c049a52 100644 --- a/cores/arduino/SerialUSB.h +++ b/cores/arduino/SerialUSB.h @@ -8,10 +8,11 @@ #include -#if defined(CONFIG_USB_DEVICE_STACK_NEXT) +#if ZARD_BOARD_HAS_SERIALUSB + #include + extern "C" struct usbd_context *usbd_init_device(usbd_msg_cb_t msg_cb); -#endif namespace arduino { @@ -34,6 +35,8 @@ class SerialUSB_ : public ZephyrSerial { return write(&data, 1); } + using Print::write; + void flush() override; protected: @@ -44,15 +47,17 @@ class SerialUSB_ : public ZephyrSerial { private: bool started = false; -#if defined(CONFIG_USB_DEVICE_STACK_NEXT) struct usbd_context *_usbd; int enable_usb_device_next(); static void usbd_next_cb(struct usbd_context *const ctx, const struct usbd_msg *msg); static int usb_disable(); -#endif }; } // namespace arduino +extern arduino::SerialUSB_ SerialUSB; + #if ZARD_FIRST_SERIAL_IS_SERIALUSB extern arduino::SerialUSB_ Serial; #endif + +#endif /* ZARD_BOARD_HAS_SERIALUSB */ diff --git a/cores/arduino/USB.cpp b/cores/arduino/USB.cpp index e152bb33c..5fb6cb165 100644 --- a/cores/arduino/USB.cpp +++ b/cores/arduino/USB.cpp @@ -10,9 +10,8 @@ #include #include -#if ZARD_FIRST_SERIAL_IS_SERIALUSB -const struct device *const usb_dev = - DEVICE_DT_GET(DT_PHANDLE_BY_IDX(DT_PATH(zephyr_user), cdc_acm_serial, 0)); +#if ZARD_BOARD_HAS_SERIALUSB +const struct device *const usb_dev = DEVICE_DT_GET(ZARD_SERIALUSB_PHANDLE); void __attribute__((weak)) _on_1200_bps() { NVIC_SystemReset(); @@ -26,11 +25,10 @@ void arduino::SerialUSB_::baudChangeHandler(const struct device *dev, uint32_t r } } -#if defined(CONFIG_USB_DEVICE_STACK_NEXT) int arduino::SerialUSB_::usb_disable() { // To avoid Cannot perform port reset: 1200-bps touch: setting DTR to OFF: protocol error k_sleep(K_MSEC(100)); - return usbd_disable(Serial._usbd); + return usbd_disable(SerialUSB._usbd); } void arduino::SerialUSB_::usbd_next_cb(struct usbd_context *const ctx, const struct usbd_msg *msg) { @@ -46,8 +44,8 @@ void arduino::SerialUSB_::usbd_next_cb(struct usbd_context *const ctx, const str if (msg->type == USBD_MSG_CDC_ACM_LINE_CODING) { uint32_t baudrate; - uart_line_ctrl_get(Serial.uart, UART_LINE_CTRL_BAUD_RATE, &baudrate); - Serial.baudChangeHandler(nullptr, baudrate); + uart_line_ctrl_get(SerialUSB.uart, UART_LINE_CTRL_BAUD_RATE, &baudrate); + SerialUSB.baudChangeHandler(nullptr, baudrate); } } @@ -67,7 +65,6 @@ int arduino::SerialUSB_::enable_usb_device_next(void) { } return 0; } -#endif /* defined(CONFIG_USB_DEVICE_STACK_NEXT) */ void arduino::SerialUSB_::begin(unsigned long baudrate, uint16_t config) { if (!started) { @@ -87,23 +84,33 @@ void arduino::SerialUSB_::begin(unsigned long baudrate, uint16_t config) { } arduino::SerialUSB_::operator bool() { - uart_line_ctrl_get(uart, UART_LINE_CTRL_DTR, &dtr); - return dtr; + int old_dtr = dtr; + int ret = uart_line_ctrl_get(uart, UART_LINE_CTRL_DTR, &dtr); + ret = dtr && (ret == 0); + if (ret && old_dtr == 0) { + k_sleep(K_MSEC(1)); + } + return ret; } size_t arduino::SerialUSB_::write(const uint8_t *buffer, size_t size) { - if (!Serial) { + if (!SerialUSB) { return 0; } return arduino::ZephyrSerial::write(buffer, size); } void arduino::SerialUSB_::flush() { - if (!Serial) { + if (!SerialUSB) { return; } arduino::ZephyrSerial::flush(); } -arduino::SerialUSB_ Serial(usb_dev); +arduino::SerialUSB_ SerialUSB(usb_dev); + +#if ZARD_FIRST_SERIAL_IS_SERIALUSB +extern arduino::SerialUSB_ Serial [[gnu::alias("SerialUSB")]]; +#endif + #endif diff --git a/cores/arduino/abi.cpp b/cores/arduino/abi.cpp index 457b56732..60e98d102 100644 --- a/cores/arduino/abi.cpp +++ b/cores/arduino/abi.cpp @@ -23,4 +23,130 @@ int __cxa_atexit(void (*func)(void *), void *arg, void *dso_handle) { return 0; } +// C++ exception ABI / unwinder stubs. +// +// Sketches are compiled with -fno-exceptions, but the precompiled libstdc++ +// archive is not. Using std::string / std::error_code pulls in objects such as +// functexcept.o and system_error.o whose throw and cleanup paths still reference +// the full exception-handling and stack-unwinding ABI: +// - libstdc++ eh_*.o (__cxa_throw, __cxa_*catch, __cxa_*cleanup, ...) +// - libgcc unwinder (_Unwind_*, __aeabi_unwind_cpp_pr0/1/2, +// __gnu_Unwind_Find_exidx, __exidx_start/__exidx_end) +// +// A sketch is loaded as an llext (relocatable object) at runtime. The llext +// loader resolves undefined symbols against the firmware's export table; the +// firmware exports none of the unwinder symbols above, so loading fails with +// "Undefined symbol ... __gnu_Unwind_Find_exidx / __exidx_start / __exidx_end". +// Pulling the libgcc unwinder in (e.g. via -lgcc) does not help: its personality +// routines reference __gnu_Unwind_Find_exidx and the linker-defined __exidx_* +// bounds, which simply do not exist in a relocatable llext. +// +// Providing strong stubs for the whole exception ABI here keeps every libstdc++ +// eh_*.o and the libgcc unwinder OUT of the sketch, so the llext is fully +// self-contained and links/loads cleanly. None of these can be reached at +// runtime because no exception is ever actually thrown; abort() guards misuse. +// +// The link recipe forces this object to be pulled before libstdc++ is scanned +// (see -Wl,-u,__cxa_allocate_exception in platform.txt) so these definitions win +// over the archive members instead of colliding with them. +void *__cxa_allocate_exception(size_t thrown_size) { + (void)thrown_size; // unused + abort(); +} + +void __cxa_free_exception(void *thrown_exception) { + (void)thrown_exception; // unused +} + +void *__cxa_allocate_dependent_exception(void) { + abort(); +} + +void __cxa_free_dependent_exception(void *dependent_exception) { + (void)dependent_exception; // unused +} + +void __cxa_throw(void *thrown_exception, void *tinfo, void (*dest)(void *)) { + (void)thrown_exception; + (void)tinfo; + (void)dest; + abort(); +} + +void __cxa_rethrow(void) { + abort(); +} + +void *__cxa_begin_catch(void *exc) { + (void)exc; + abort(); +} + +void __cxa_end_catch(void) { +} + +void __cxa_begin_cleanup(void *exc) { + (void)exc; + abort(); +} + +void __cxa_end_cleanup(void) { + abort(); +} + +void *__cxa_type_match(void *exc, const void *catch_type, bool is_ref, void **obj) { + (void)exc; + (void)catch_type; + (void)is_ref; + (void)obj; + abort(); +} + +void __cxa_call_unexpected(void *exc) { + (void)exc; + abort(); +} + +int __gxx_personality_v0(int version, int actions, unsigned exc_class, void *exc, void *ctx) { + (void)version; + (void)actions; + (void)exc_class; + (void)exc; + (void)ctx; + abort(); +} + +void _Unwind_Resume(void *exc) { + (void)exc; + abort(); +} + +// ARM EABI compact-model personality routines referenced by the exception +// tables of the libstdc++ objects above. Stubbing them prevents the libgcc +// unwinder (and its __gnu_Unwind_Find_exidx / __exidx_* references) from being +// pulled in. +int __aeabi_unwind_cpp_pr0(void) { + abort(); +} + +int __aeabi_unwind_cpp_pr1(void) { + abort(); +} + +int __aeabi_unwind_cpp_pr2(void) { + abort(); +} + +// picolibc strerror() hook. std::error_code::message() (system_error.o) calls +// __xpg_strerror_r() which references the weak picolibc hook _user_strerror. +// It is not provided by the firmware, so define it here (no custom error +// strings -> fall back to picolibc's built-in table) to keep the llext +// self-contained. +char *_user_strerror(int errnum, int internal, int *errptr) { + (void)errnum; + (void)internal; + (void)errptr; + return 0; +} + } /* extern "C" */ diff --git a/cores/arduino/api b/cores/arduino/api deleted file mode 120000 index d32ec6953..000000000 --- a/cores/arduino/api +++ /dev/null @@ -1 +0,0 @@ -../../../modules/lib/ArduinoCore-API/api/ \ No newline at end of file diff --git a/cores/arduino/llext_wrappers.c b/cores/arduino/llext_wrappers.c index 8a307ead6..0ea624939 100644 --- a/cores/arduino/llext_wrappers.c +++ b/cores/arduino/llext_wrappers.c @@ -12,6 +12,7 @@ #include #include +#include /* ret func(void) */ #define W0(ret, name) \ @@ -118,6 +119,10 @@ W3(void *, memchr, const void *, int, size_t) W1(char *, strdup, const char *) W4(void *, memmem, const void *, size_t, const void *, size_t); +/* XSI strerror_r (picolibc symbol). Keeps libstdc++'s std::error_code + * message() path from bundling _strerror_r and its message table per sketch. */ +W3(int, __xpg_strerror_r, int, char *, size_t) + /* stdlib.h - conversion */ W2(double, strtod, const char *, char **) W3(long, strtol, const char *, char **, int) @@ -195,8 +200,17 @@ VN(__aeabi_dcmple) VN(__aeabi_dcmpgt) VN(__aeabi_dcmpge) VN(__aeabi_dcmpun) +/* float arithmetic */ +VN(__aeabi_fadd) +VN(__aeabi_fsub) +VN(__aeabi_fmul) +VN(__aeabi_fdiv) /* float comparisons */ +VN(__aeabi_fcmpeq) +VN(__aeabi_fcmplt) VN(__aeabi_fcmple) +VN(__aeabi_fcmpgt) +VN(__aeabi_fcmpge) VN(__aeabi_fcmpun) /* double <-> integer conversions */ VN(__aeabi_d2iz) @@ -209,6 +223,10 @@ VN(__aeabi_ul2d) /* float <-> double / integer conversions */ VN(__aeabi_d2f) VN(__aeabi_f2d) +VN(__aeabi_i2f) +VN(__aeabi_ui2f) +VN(__aeabi_f2iz) +VN(__aeabi_f2uiz) VN(__aeabi_l2f) VN(__aeabi_ul2f) /* integer division */ @@ -235,6 +253,74 @@ W1(int, puts, const char *) W1(int, putchar, int) W4(int, vsnprintf, char *, size_t, const char *, va_list) +/* time.h */ +W2(int, gettimeofday, struct timeval *, void *) + +#ifdef CONFIG_STACK_CANARIES +/* zephyr/posix/unistd.h */ +W2(int, getentropy, void *, size_t) +#endif + +/* + * Variadic stdio functions cannot be expressed with the tail-call wrapper + * macros above. Provide explicit trampolines that capture the argument list + * and forward to the corresponding v* form, which is bound at load time. + * Defining these strong symbols in the core makes them win over the archive + * members during the sketch link, keeping picolibc's sscanf/vfscanf/vfprintf + * and the __atod_engine/__atof_engine float helpers out of the sketch llext. + */ +extern int __real_vprintf(const char *, va_list); + +int printf(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + int ret = __real_vprintf(fmt, ap); + va_end(ap); + return ret; +} + +extern int __real_vsprintf(char *, const char *, va_list); + +int sprintf(char *str, const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + int ret = __real_vsprintf(str, fmt, ap); + va_end(ap); + return ret; +} + +int snprintf(char *str, size_t size, const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + int ret = __real_vsnprintf(str, size, fmt, ap); + va_end(ap); + return ret; +} + +extern int __real_vsscanf(const char *, const char *, va_list); + +int sscanf(const char *str, const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + int ret = __real_vsscanf(str, fmt, ap); + va_end(ap); + return ret; +} + +/* Direct va_list variants: same forwarding, exposed under their own names so a + * sketch calling them directly does not pull picolibc's vfprintf/vfscanf. */ +int vprintf(const char *fmt, va_list ap) { + return __real_vprintf(fmt, ap); +} + +int vsprintf(char *str, const char *fmt, va_list ap) { + return __real_vsprintf(str, fmt, ap); +} + +int vsscanf(const char *str, const char *fmt, va_list ap) { + return __real_vsscanf(str, fmt, ap); +} + /* stdlib.h - atexit */ typedef void (*__atexit_fn)(void); W1(int, atexit, __atexit_fn) diff --git a/cores/arduino/main.cpp b/cores/arduino/main.cpp index 88bc19327..daa82b277 100644 --- a/cores/arduino/main.cpp +++ b/cores/arduino/main.cpp @@ -15,17 +15,17 @@ void start_static_threads(); #endif -// This function will be overwriten by most variants. +// This function will be overwritten by most variants. void __attribute__((weak)) initVariant(void) { } -// This function can be overwriten by one library. +// This function can be overwritten by one library. void __attribute__((weak)) __loopHook(void) { } int main(void) { -#if ZARD_FIRST_SERIAL_IS_SERIALUSB - Serial.begin(115200); +#if ZARD_BOARD_HAS_SERIALUSB + SerialUSB.begin(115200); #endif initVariant(); diff --git a/cores/arduino/usb_device_descriptor.c b/cores/arduino/usb_device_descriptor.c index 646bd84d7..afddf44e6 100644 --- a/cores/arduino/usb_device_descriptor.c +++ b/cores/arduino/usb_device_descriptor.c @@ -11,12 +11,19 @@ #include #include +#include +#if __has_include() +#include +#endif #ifdef CONFIG_USB_DEVICE_STACK_NEXT /* By default, do not register the USB DFU class DFU mode instance. */ static const char *const blocklist[] = { "dfu_dfu", +#if defined(LOADER_PROVIDES_EXTRA_USB_CLASSES) + "cdc_acm_0", +#endif NULL, }; @@ -32,8 +39,8 @@ USBD_DESC_PRODUCT_DEFINE(sample_product, CONFIG_USB_DEVICE_PRODUCT); USBD_DESC_SERIAL_NUMBER_DEFINE(sample_sn); /* doc string instantiation end */ -USBD_DESC_CONFIG_DEFINE(fs_cfg_desc, "FS Configuration"); -USBD_DESC_CONFIG_DEFINE(hs_cfg_desc, "HS Configuration"); +USBD_DESC_CONFIG_DEFINE(fs_cfg_desc, APP_VERSION_EXTENDED_STRING); +USBD_DESC_CONFIG_DEFINE(hs_cfg_desc, APP_VERSION_EXTENDED_STRING); /* doc configuration instantiation start */ static const uint8_t attributes = 0; @@ -73,6 +80,19 @@ static void sample_fix_code_triple(struct usbd_context *uds_ctx, const enum usbd } } +#if defined(LOADER_PROVIDES_EXTRA_USB_CLASSES) +static int register_cdc_acm_0(struct usbd_context *const uds_ctx, const enum usbd_speed speed) { + int err; + + err = usbd_register_class(&usbd, "cdc_acm_0", speed, 1); + if (err) { + return err; + } + + return usbd_device_set_code_triple(uds_ctx, speed, USB_BCC_MISCELLANEOUS, 0x02, 0x01); +} +#endif + struct usbd_context *usbd_setup_device(usbd_msg_cb_t msg_cb) { int err; @@ -109,6 +129,10 @@ struct usbd_context *usbd_setup_device(usbd_msg_cb_t msg_cb) { return NULL; } +#if defined(LOADER_PROVIDES_EXTRA_USB_CLASSES) + register_cdc_acm_0(&usbd, USBD_SPEED_HS); +#endif + sample_fix_code_triple(&usbd, USBD_SPEED_HS); } @@ -126,6 +150,10 @@ struct usbd_context *usbd_setup_device(usbd_msg_cb_t msg_cb) { } /* doc functions register end */ +#if defined(LOADER_PROVIDES_EXTRA_USB_CLASSES) + register_cdc_acm_0(&usbd, USBD_SPEED_FS); +#endif + sample_fix_code_triple(&usbd, USBD_SPEED_FS); if (msg_cb != NULL) { diff --git a/cores/arduino/wiring_analog.cpp b/cores/arduino/wiring_analog.cpp index 961f098f5..3c7d06b01 100644 --- a/cores/arduino/wiring_analog.cpp +++ b/cores/arduino/wiring_analog.cpp @@ -97,15 +97,12 @@ static const struct dac_channel_cfg dac_ch_cfg[] = { int _analog_write_resolution = 8; #endif -// Note: We can not update the arduino_adc structure as it is read only... -static int read_resolution = 10; +} // anonymous namespace uint32_t map64(uint32_t x, uint32_t in_min, uint32_t in_max, uint32_t out_min, uint32_t out_max) { return ((uint64_t)(x - in_min) * (out_max - out_min) / (in_max - in_min) + out_min); } -} // anonymous namespace - #if defined(CONFIG_DAC) || defined(CONFIG_PWM) void analogWriteResolution(int bits) { _analog_write_resolution = CLAMP(bits, 1, 31); @@ -184,6 +181,9 @@ void analogWrite(enum dacPins dacName, int value) { #ifdef CONFIG_ADC +// Note: We can not update the arduino_adc structure as it is read only... +static int read_resolution = 10; + void __attribute__((weak)) analogReference(uint8_t mode) { /* * The Arduino API not clearly defined what means of diff --git a/cores/arduino/zephyrPinctrl.cpp b/cores/arduino/zephyrPinctrl.cpp index 31a42880a..d1500c5e0 100644 --- a/cores/arduino/zephyrPinctrl.cpp +++ b/cores/arduino/zephyrPinctrl.cpp @@ -30,12 +30,13 @@ namespace arduino { * Map deferred-init peripherals and zephyr,console (that is not deferred) with their * pinctrl configuration from devicetree. */ -#define NODE_SELECTED(node_id) \ - UTIL_OR(DT_PROP(node_id, zephyr_deferred_init), \ - DT_SAME_NODE(node_id, DT_CHOSEN(zephyr_console))) +#define PINCTRL_NODE_SELECTED(node_id) \ + UTIL_AND(DT_NODE_HAS_PROP(node_id, pinctrl_0), \ + UTIL_OR(DT_PROP(node_id, zephyr_deferred_init), \ + DT_SAME_NODE(node_id, DT_CHOSEN(zephyr_console)))) #define PINCTRL_DEFINE_IF_SELECTED(node_id) \ - COND_CODE_1(NODE_SELECTED(node_id), (PINCTRL_DT_DEFINE(node_id);), ()) + COND_CODE_1(PINCTRL_NODE_SELECTED(node_id), (PINCTRL_DT_DEFINE(node_id);), ()) DT_FOREACH_STATUS_OKAY_NODE(PINCTRL_DEFINE_IF_SELECTED) @@ -46,7 +47,7 @@ struct pinctrl_map_entry { #define PINCTRL_MAP_ENTRY(node_id) {DEVICE_DT_GET(node_id), PINCTRL_DT_DEV_CONFIG_GET(node_id)}, #define PINCTRL_MAP_ENTRY_IF_PRESENT(node_id) \ - COND_CODE_1(NODE_SELECTED(node_id), (PINCTRL_MAP_ENTRY(node_id)), ()) + COND_CODE_1(PINCTRL_NODE_SELECTED(node_id), (PINCTRL_MAP_ENTRY(node_id)), ()) static const struct pinctrl_map_entry pinctrl_map[] = { DT_FOREACH_STATUS_OKAY_NODE(PINCTRL_MAP_ENTRY_IF_PRESENT){NULL, NULL}, @@ -192,7 +193,7 @@ int init_dev_apply_pinctrl(const struct device *dev) { * exports. */ #define DECLARE_DEVICE_WEAK(node_id) \ - COND_CODE_1(NODE_SELECTED(node_id), \ + COND_CODE_1(PINCTRL_NODE_SELECTED(node_id), \ (extern const struct device DEVICE_DT_NAME_GET(node_id) __attribute__((weak));), ()) DT_FOREACH_STATUS_OKAY_NODE(DECLARE_DEVICE_WEAK) diff --git a/cores/arduino/zephyrSerial.cpp b/cores/arduino/zephyrSerial.cpp index 112d66500..84ab1c3f0 100644 --- a/cores/arduino/zephyrSerial.cpp +++ b/cores/arduino/zephyrSerial.cpp @@ -86,11 +86,7 @@ void arduino::ZephyrSerial::IrqHandler() { k_sem_take(&rx.sem, K_NO_WAIT); while (uart_irq_rx_ready(uart) && ((length = uart_fifo_read(uart, buf, sizeof(buf))) > 0)) { length = min(sizeof(buf), static_cast(length)); - ret = ring_buf_put(&rx.ringbuf, &buf[0], length); - - if (ret < 0) { - break; - } + ring_buf_put(&rx.ringbuf, &buf[0], length); } k_sem_give(&rx.sem); @@ -166,9 +162,6 @@ size_t arduino::ZephyrSerial::write(const uint8_t *buffer, size_t size) { k_sem_take(&tx.sem, K_FOREVER); auto ret = ring_buf_put(&tx.ringbuf, &buffer[idx], size - idx); k_sem_give(&tx.sem); - if (ret < 0) { - return 0; - } idx += ret; if (ret == 0) { uart_irq_tx_enable(uart); diff --git a/cores/arduino/zephyrSerial.h b/cores/arduino/zephyrSerial.h index b4e771e16..e2a52c907 100644 --- a/cores/arduino/zephyrSerial.h +++ b/cores/arduino/zephyrSerial.h @@ -121,18 +121,41 @@ class ZephyrSerial : public HardwareSerial { #define ZARD_SERIAL_INDEXOF(node) \ DT_FOREACH_PROP_ELEM_VARGS(DT_PATH(zephyr_user), serials, ZARD_SERIAL_MATCH, node) +/* Get the name of the Serial object associated with a given node. If the node + * is not in the 'serials' array, then check if it is the SerialUSB node. + */ +#define ZARD_SERIAL_NAME_BY_NODE(node) \ + COND_CODE_0(IS_EMPTY(ZARD_SERIAL_INDEXOF(node)), \ + (ZARD_SERIAL_NAME(ZARD_SERIAL_INDEXOF(node))), \ + (COND_CODE_1(DT_SAME_NODE(node, ZARD_SERIALUSB_PHANDLE), \ + (SerialUSB), (unkown Serial object)))) + /* Serial object associated with the Zephyr console. */ -#define ARDUINO_CONSOLE_SERIAL ZARD_SERIAL_NAME(ZARD_SERIAL_INDEXOF(DT_CHOSEN(zephyr_console))) +#define ARDUINO_CONSOLE_SERIAL ZARD_SERIAL_NAME_BY_NODE(DT_CHOSEN(zephyr_console)) /* Serial object associated with the first HW serial (usually on D0/D1). */ #define ARDUINO_HARDWARE_SERIAL ZARD_SERIAL_NAME(0) +#if DT_NODE_HAS_PROP(DT_PATH(zephyr_user), arduino_router_serial) +/* If the board has an arduino,router-serial node, and the currently used + * library has the support, then the 'Serial' object is actually the same as + * the Monitor object in the Arduino_RouterBridge library. + */ +#define ZARD_FIRST_SERIAL_IS_ARDUINO_ROUTER 1 +#define ARDUINO_ROUTER_PHANDLE DT_PROP(DT_PATH(zephyr_user), arduino_router_serial) +#define ARDUINO_ROUTER_SERIAL ZARD_SERIAL_NAME_BY_NODE(ARDUINO_ROUTER_PHANDLE) +#endif + #if DT_NODE_HAS_PROP(DT_PATH(zephyr_user), cdc_acm_serial) /* Devicetree requires a SerialUSB object for 'Serial'. */ -#define ZARD_SKIP_FIRST_SERIAL 1 -#if (CONFIG_USB_CDC_ACM || CONFIG_USBD_CDC_ACM_CLASS) +#if CONFIG_USBD_CDC_ACM_CLASS /* SerialUSB can be compiled in the project. */ +#define ZARD_BOARD_HAS_SERIALUSB 1 +#define ZARD_SERIALUSB_PHANDLE DT_PROP(DT_PATH(zephyr_user), cdc_acm_serial) +/* Router takes precedence as 'Serial' when both are defined. */ +#if !ZARD_FIRST_SERIAL_IS_ARDUINO_ROUTER #define ZARD_FIRST_SERIAL_IS_SERIALUSB 1 +#endif #else /* SerialUSB is required but no driver was enabled for the USB CDC ACM device. * Define a stub Serial object to avoid build errors. @@ -141,15 +164,10 @@ class ZephyrSerial : public HardwareSerial { #endif #endif -#if DT_NODE_HAS_PROP(DT_PATH(zephyr_user), arduino_router_serial) -/* If the board has an arduino,router-serial node, and the currently used - * library has the support, then the 'Serial' object is actually the same as - * the Monitor object in the Arduino_RouterBridge library. - */ -#define ZARD_SKIP_FIRST_SERIAL 1 -#define ZARD_FIRST_SERIAL_IS_ARDUINO_ROUTER 1 -#define ARDUINO_ROUTER_PHANDLE DT_PROP(DT_PATH(zephyr_user), arduino_router_serial) -#define ARDUINO_ROUTER_SERIAL ZARD_SERIAL_NAME(ZARD_SERIAL_INDEXOF(ARDUINO_ROUTER_PHANDLE)) +/* If one of the above groups match, they will define the first Serial object */ +#if ZARD_FIRST_SERIAL_IS_SERIALUSB || ZARD_FIRST_SERIAL_IS_ARDUINO_ROUTER || \ + ZARD_FIRST_SERIAL_IS_STUB +#define ZARD_SKIP_FIRST_SERIAL 1 #endif /* Name of a Serial object for a given index. */ diff --git a/cores/arduino/zephyr_sketch_header.h b/cores/arduino/zephyr_sketch_header.h new file mode 100644 index 000000000..cd5adf79b --- /dev/null +++ b/cores/arduino/zephyr_sketch_header.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) Arduino s.r.l. and/or its affiliated companies + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +struct sketch_header_v1 { + uint8_t ver; // @ 0x07 + uint32_t len; // @ 0x08 + uint16_t magic; // @ 0x0c + uint8_t flags; // @ 0x0e +} __attribute__((packed)); + +#define SKETCH_HEADER_LEN 16 + +#define SKETCH_FLAG_DEBUG 0x01 +#define SKETCH_FLAG_LINKED 0x02 +#define SKETCH_FLAG_IMMEDIATE 0x04 +#define SKETCH_FLAG_WAIT_FOR_APP 0x08 + +#define SKETCH_VERSION_LATEST 0x1 +#define SKETCH_MAGIC 0x2341 + +int sketch_header_v1_verify(const struct sketch_header_v1 *hdr); diff --git a/documentation/arduino_libs.md b/documentation/arduino_libs.md index 9118575c4..779161598 100644 --- a/documentation/arduino_libs.md +++ b/documentation/arduino_libs.md @@ -18,11 +18,11 @@ Let's assume the structure of your application for simplicity to be, Paste your library's source files like ``ADXL345.h`` and ``ADXL345.cpp`` in your project's src folder. ## Update CMakeLists -As we can see, there is a Top-level CMakelists which needs to be updated with your external library's source files. +As we can see, there is a Top-level CMakeLists.txt which needs to be updated with your external library's source files. For example, we add `target_sources(app PRIVATE src/ADXL345.cpp)` to CMakeLists.txt. ## Update main.cpp -Finally, paste your required code using the library in ``main.c`` and include that library's header using +Finally, paste your required code using the library in ``main.cpp`` and include that library's header using ```c #include "ADXL345.h" ``` diff --git a/documentation/dynamic_pinmux.md b/documentation/dynamic_pinmux.md index 350d80ec8..993057f91 100644 --- a/documentation/dynamic_pinmux.md +++ b/documentation/dynamic_pinmux.md @@ -29,7 +29,7 @@ PINCTRL_STATE_SLEEP = 1 PINCTRL_STATE_PRIV_START = 2 ``` -Custom state name arduino is defined in devicetree and mapped to PINCTRL_STATE_ARDUINO. +The custom state name `arduino` is defined in devicetree and mapped to `PINCTRL_STATE_ARDUINO`. Typical mapping: @@ -37,67 +37,67 @@ Typical mapping: - sleep: optional low-power/disconnected state (not used) - arduino: custom channel list used for per-pin restore (mainly ADC/PWM/DAC) -## Runtime Infrastructure in zephyrPinctrl.cpp +## Runtime Infrastructure in `zephyrPinctrl.cpp` The core builds a runtime map between devices and pinctrl configs. -Selection rule for nodes included in the map is node is zephyr,deferred-init. -Because the Zephyr logging UART is intentionally not marked with zephyr,deferred-init for easier debugging, it is added to the pinctrl map via a dedicated mapping entry. +The selection rule for nodes included in the map is that the node has `zephyr,deferred-init`. +Because the Zephyr logging UART is intentionally not marked with `zephyr,deferred-init` for easier debugging, it is added to the pinctrl map via a dedicated mapping entry. The map is then used by APIs: -- init_dev_apply_pinctrl(dev) -- init_dev_apply_channel_pinctrl(dev, state_pin_idx) +- `init_dev_apply_pinctrl(dev)` +- `init_dev_apply_channel_pinctrl(dev, state_pin_idx)` If a device has no known pinctrl config in the map: -- init_dev_apply_channel_pinctrl(...) returns -ENOTSUP, -- init_dev_apply_pinctrl(...) still initializes the device/dependencies but skips pinctrl apply for entries +- `init_dev_apply_channel_pinctrl(...)` returns `-ENOTSUP`, +- `init_dev_apply_pinctrl(...)` still initializes the device/dependencies but skips pinctrl apply for entries not present in the map. ## Dynamic Pinmux APIs and Runtime Flow -### init_dev_apply_pinctrl(dev) +### `init_dev_apply_pinctrl(dev)` Purpose: initialize the target device and its dependencies, then apply the whole pinctrl -PINCTRL_STATE_DEFAULT state where pinctrl configuration is known. +`PINCTRL_STATE_DEFAULT` state where pinctrl configuration is known. Used for full peripheral remux, to restore the default state. -SPI (SPI.begin), I2C (Wire.begin) and UART (ZephyrSerial::begin) calls init_dev_apply_pinctrl(dev), without taking care of device readiness/init. +SPI (`SPI.begin`), I2C (`Wire.begin`) and UART (`ZephyrSerial::begin`) call `init_dev_apply_pinctrl(dev)`, without taking care of device readiness/init. Behavior details: - recursively initializes required dependencies first, - initializes each device on first use if needed, -- applies PINCTRL_STATE_DEFAULT from known device pinctrl configs, -- treats -ENOENT from pinctrl_apply_state as non-fatal (for empty/default-less states). +- applies `PINCTRL_STATE_DEFAULT` from known device pinctrl configs, +- treats `-ENOENT` from `pinctrl_apply_state` as non-fatal (for empty/default-less states). -### init_dev_apply_channel_pinctrl(dev, state_pin_idx) +### `init_dev_apply_channel_pinctrl(dev, state_pin_idx)` -Purpose: apply only one pin from the device arduino state. +Purpose: apply only one pin from the device `arduino` state. -Used for channel-based peripherals (ADC/PWM/DAC) in the analogRead and analogWrite functions, allowing a single channel can be remuxed without touching sibling channels. +Used for channel-based peripherals (ADC/PWM/DAC) in the `analogRead` and `analogWrite` functions, allowing a single channel to be remuxed without touching sibling channels. Behavior details: - initializes the device on first use if needed, -- resolves PINCTRL_STATE_ARDUINO, -- applies only one pin entry (state_pin_idx), with bounds checking. +- resolves `PINCTRL_STATE_ARDUINO`, +- applies only one pin entry (`state_pin_idx`), with bounds checking. Implementation pattern: - resolve analog/pwm index from pin, -- compute a per-device ordinal index with state_pin_index_from_spec_index(arduino_adc/arduino_pwm, idx), -- call init_dev_apply_channel_pinctrl(dev, per_dev_idx), +- compute a per-device ordinal index with `state_pin_index_from_spec_index(arduino_adc/arduino_pwm, idx)`, +- call `init_dev_apply_channel_pinctrl(dev, per_dev_idx)`, -How state_pin_index_from_spec_index works: +How `state_pin_index_from_spec_index` works: -- spec_idx is the global position in the zephyr,user array (io-channels for ADC or pwms for PWM). For example, A1/D1 maps to spec_idx = 1 (second entry); -- state_pin_index_from_spec_index(...) scans previous entries [0..spec_idx-1] and counts only those that reference the same device (i.e. the same PWM/ADC controller instance, such as pwm4); -- the resulting count is the per-device ordinal (0, 1, 2, ...) used as state_pin_idx in the device arduino pinctrl state. +- `spec_idx` is the global position in the `zephyr,user` array (`io-channels` for ADC or `pwms` for PWM). For example, A1/D1 maps to `spec_idx = 1` (second entry); +- `state_pin_index_from_spec_index(...)` scans previous entries `[0..spec_idx-1]` and counts only those that reference the same device (i.e. the same PWM/ADC controller instance, such as `pwm4`); +- the resulting count is the per-device ordinal (0, 1, 2, ...) used as `state_pin_idx` in the device `arduino` pinctrl state. -Example (GIGA PWM list in zephyr,user): +Example (GIGA PWM list in `zephyr,user`): ```dts pwms = <&pwm1 3 PWM_HZ(12000000) PWM_POLARITY_NORMAL>, @@ -116,14 +116,14 @@ pwms = <&pwm1 3 PWM_HZ(12000000) PWM_POLARITY_NORMAL>, <&pwm1 1 PWM_HZ(5000) PWM_POLARITY_NORMAL>; ``` -If analogWrite() resolves to spec_idx = 8 (9th entry, `&pwm4 4`): +If `analogWrite()` resolves to `spec_idx = 8` (9th entry, `&pwm4 4`): -- device = pwm4 -- previous pwm4 entries are at spec_idx 5 and 7 +- device = `pwm4` +- previous `pwm4` entries are at `spec_idx` 5 and 7 - count of previous same-device entries = 2 -- state_pin_idx = 2 (zero-based) +- `state_pin_idx = 2` (zero-based) -Then init_dev_apply_channel_pinctrl(pwm4, 2) selects the 3rd entry of pwm4 `pinctrl-2`. +Then `init_dev_apply_channel_pinctrl(pwm4, 2)` selects the 3rd entry of `pwm4` `pinctrl-2`. ```dts pwm4: pwm { @@ -140,7 +140,7 @@ In this example, the selected pin is `tim4_ch4_pb9`. Effect: -- only the requested ADC pad or PWM pad is remuxed to analog function. +- only the requested ADC pad or PWM pad is remuxed to its target peripheral function. ## Devicetree Requirements @@ -158,17 +158,17 @@ Use deferred init and provide usual default/sleep states. }; ``` -Same pattern applies to i2cX and uartX. +Same pattern applies to `i2cX` and `uartX`. Note: the runtime map explicitly includes the node selected as `zephyr,console` even when it is -not deferred-init. +not `deferred-init`. -Recommendation: keep the UART used for early logs available from boot; if it is deferred-init, +Recommendation: keep the UART used for early logs available from boot; if it is `deferred-init`, early error logs generated during system startup or sketch load may not be visible. ### For channel-based ADC/PWM/DAC nodes -Use deferred init and keep channel pads in pinctrl-2 and name it arduino. +Use deferred init and keep channel pads in `pinctrl-2` and name it `arduino`. ```dts &periphN { @@ -185,16 +185,16 @@ Use deferred init and keep channel pads in pinctrl-2 and name it arduino. For each ADC/PWM device, ordering must match between: -- zephyr,user spec arrays (io-channels or pwms), and -- that device arduino pin list (pinctrl-2). +- `zephyr,user` spec arrays (`io-channels` or `pwms`), and +- that device `arduino` pin list (`pinctrl-2`). -state_pin_index_from_spec_index() assumes this ordinal alignment to map a logical channel entry to the correct pin in the arduino state. +`state_pin_index_from_spec_index()` assumes this ordinal alignment to map a logical channel entry to the correct pin in the `arduino` state. ## Known limitation - Nordic nRF On Nordic nRF targets, switching a pin from PWM back to another peripheral may leave PWM still affecting that pin unless PWM is explicitly stopped first. -The reason is architectural: in the nRF pinctrl/driver model, pin routing is programmed per peripheral by writing that peripheral’s PSEL registers. Reconfiguring the new peripheral (i.e. SPI) updates SPI’s PSEL, but it does not automatically clear PWM’s PSEL ownership for the same physical pin. +The reason is architectural: in the nRF pinctrl/driver model, pin routing is programmed per peripheral by writing that peripheral’s `PSEL` registers. Reconfiguring the new peripheral (i.e. SPI) updates SPI’s `PSEL`, but it does not automatically clear PWM’s `PSEL` ownership for the same physical pin. In our current core flow, we do not keep global runtime ownership tracking across peripherals for each pin, so we cannot reliably force-release the previous owner in all cases. This limitation is relevant only when the same pin was previously configured/driven by PWM and is now being reassigned to another peripheral. diff --git a/documentation/variants.md b/documentation/variants.md index 300c6a17a..5f5ff6e60 100644 --- a/documentation/variants.md +++ b/documentation/variants.md @@ -1,10 +1,10 @@ -# Adding custom boards/ variants +# Adding custom boards/variants - Boards already supported by Zephyr can be added to the variants folder as outlined in this documentation. -- Custom boards can first by added by following the [official Zephyr porting guide](https://docs.zephyrproject.org/latest/hardware/porting/board_porting.html). +- Custom boards can first be added by following the [official Zephyr porting guide](https://docs.zephyrproject.org/latest/hardware/porting/board_porting.html). Once completed, continue here by adding a variant for your custom board. -## Suppored Boards/variants +## Supported Boards/variants - [X] Arduino Nano ble sense 33 - [X] Arduino Nano ble 33 @@ -26,7 +26,7 @@ target board. To add board support: 1. This project is structured in a way so as to isolate the variants from the core API. Thus, whenever a new board needs to be added it needs to be done in the `variants/` folder. Add a folder inside of the variants folder that matches the name of your board. -2. Add an overlay file file that match the name of the board. +2. Add an overlay file that matches the name of the board. 3. Add a `variant.h` file. An example of this structure is shown below. @@ -45,13 +45,19 @@ variants/ The Arduino API requires pin mapping definitions to use Arduino-style pin numbers (pin numbers printed on the board, not GPIO numbers). The pin-mapping node is under the `zephyr,user` node of DTS. -`digital-pin-gpios` defines digital input/output pins that is D0, D1, .., -`adc-pin-gpios` defines analog input pins that is A0, A1, ... . +`digital-pin-gpios` defines digital input/output pins, i.e. D0, D1, ..., +`adc-pin-gpios` defines analog input pins, i.e. A0, A1, etc. `pwm-pin-gpios` defines PWM pins. Each pin specifies in the form of a GPIO cell. Usually, it is in the form of `<[port] [pin-number] [flag]>`. You can also use the Arduino header node definition here. +> [!NOTE] +> For phandle-array properties like these (`digital-pin-gpios`, `adc-pin-gpios`, +> `pwm-pin-gpios`, `serials`, `i2cs`, etc.), both `<&dev1 &dev2>` (multiple entries in a +> single bracket group) and `<&dev1>, <&dev2>` (separate bracket groups joined by commas) +> are valid DTS syntax. However, `<&dev1, &dev2>` is **invalid**. + ### Overlays using previously-defined Arduino headers When an Arduino header exists in a board's in-tree DTS file it can easily be @@ -66,18 +72,17 @@ uses [the Arduino header definitions](https://github.com/zephyrproject-rtos/zeph ``` / { zephyr,user { - digital-pin-gpios = <&arduino_header 6 0>, /* Digital */ - <&arduino_header 7 0>; - ... - <&arduino_header 19 0>; - <&arduino_header 0 0>; /* Analog */ - <&arduino_header 1 0>; - ... - <&arduino_header 5 0>; - <&arduino_header 20 0>; /* SDA */ - <&arduino_header 21 0>; /* SCL */ - <&gpio0 13 GPIO_ACTIVE_LOW>; /* LED0 */ - }; + digital-pin-gpios = <&arduino_header 6 0>, /* Digital */ + <&arduino_header 7 0>, + ... + <&arduino_header 19 0>, + <&arduino_header 0 0>, /* Analog */ + <&arduino_header 1 0>, + ... + <&arduino_header 5 0>, + <&arduino_header 20 0>, /* SDA */ + <&arduino_header 21 0>, /* SCL */ + <&gpio0 13 GPIO_ACTIVE_LOW>; /* LED0 */ }; }; ``` @@ -85,15 +90,15 @@ uses [the Arduino header definitions](https://github.com/zephyrproject-rtos/zeph ### Configure Serial devices The `serials` node defines the Serial devices to use. -It instantiate the `Serial` with the UART device that contained in the node. -Also instantiate as `Serial1`, `Serial2`, .. `SerialN` with the devices that is -after the second in the case of the array contains plural devices. +It instantiates `Serial` with the UART device contained in the node. +Also instantiates `Serial1`, `Serial2`, ... `SerialN` with the devices +after the first when the array contains more than one device. -If the `serials` node is not defined, Use the node labeled `arduino-serial`. +If the `serials` node is not defined, use the node labeled `arduino-serial`. Boards with Arduino-shield style connectors usually label `arduino-serial` for UART port exposed in header or frequently used UART port. -If even 'arduino_serial' does not define, it uses the stub implementation +If even `arduino_serial` is not defined, it uses the stub implementation that redirects to printk(). The following example instantiates `Serial` and `Serial1` with each `uart0` and `uart1`. @@ -101,7 +106,7 @@ The following example instantiates `Serial` and `Serial1` with each `uart0` and ``` / { zephyr,user { - serials = <&uart0, &uart1>; + serials = <&uart0>, <&uart1>; }; }; ``` @@ -109,20 +114,20 @@ The following example instantiates `Serial` and `Serial1` with each `uart0` and ### Configure I2C devices The `i2cs` node defines the I2C devices to use. -It instantiate the `Wire` with the i2c device that contained in the node. -Also instantiate as `Wire1`, `Wire2`, .. `WireN` with the devices -that is after the second in the case of the array contains plural devices. +It instantiates `Wire` with the i2c device contained in the node. +Also instantiates `Wire1`, `Wire2`, ... `WireN` with the devices +after the first when the array contains more than one device. -If the `i2cs` node is not defined, Use the node labeled `arduino-i2c`. +If the `i2cs` node is not defined, use the node labeled `arduino-i2c`. Boards with Arduino-shield style connectors usually label `arduino-i2c` to i2c exposed in the connector. -The following example instantiates `Wire` and `Wire2` with each `i2c0` and `i2c1`. +The following example instantiates `Wire` and `Wire1` with each `i2c0` and `i2c1`. ``` / { zephyr,user { - i2cs = <&i2c0, &i2c1>; + i2cs = <&i2c0 &i2c1>; }; }; ``` @@ -135,16 +140,15 @@ array to find the index of the pin. The node is phandle-array, which uses the format same as `digital-pin-gpios`. -It set the digital pin number to the `LED_BUILTIN` if found the pin -that defined in `builtin-led-gpios` from `digital-pin-gpios`. +It sets the digital pin number to `LED_BUILTIN` if the pin +defined in `builtin-led-gpios` is found in `digital-pin-gpios`. -If the `builtin-led-gpios` is not defined, Use the node aliased as `led0` +If the `builtin-led-gpios` is not defined, use the node aliased as `led0` to define `LED_BUILTIN`. -The `LED_BUILTIN` does not define here if it has not found both nodes or -defined `LED_BUILTIN` already. +`LED_BUILTIN` is not defined if neither node is found or `LED_BUILTIN` is already defined. -For example, in the case of the 13th digital pins connected to the onboard LED, +For example, in the case of the 13th digital pin connected to the onboard LED, define `builtin-led-gpios` as follows. ``` @@ -160,7 +164,7 @@ define `builtin-led-gpios` as follows. You can see in the example above that there is no mapping for `LED0` in the board's Arduino header definition so it has been added using the Zephyr `gpios` syntax (port, pin, flags). When creating an overlay file that doesn't have an -Arduino header defined, you should follow this syntax for adding all pins +Arduino header defined, you should follow this syntax for adding all pins. ### You control the pin mapping @@ -168,7 +172,7 @@ Zephyr [chooses to map Arduino headers beginning with the Analog pins](https://docs.zephyrproject.org/latest/build/dts/api/bindings/gpio/arduino-header-r3.html), but the overlay file example above begins with the digital pins. This is to match user -expectation that issuing `pinMode(0,OUTPUT);` should control digital pin 0 (and +expectations that issuing `pinMode(0,OUTPUT);` should control digital pin 0 (and not pin 6). In the same way, the Analog 0 pin was mapped to D14 as this is likely what a shield made for the Arduino Uno R3 header would expect. diff --git a/extra/artifacts/_common.json b/extra/artifacts/_common.json index 9d9723469..d4f3e8542 100644 --- a/extra/artifacts/_common.json +++ b/extra/artifacts/_common.json @@ -16,7 +16,7 @@ { "packager": "zephyr", "name": "arm-zephyr-eabi", - "version": "0.16.8" + "version": "1.0.1" }, { "packager": "arduino", @@ -31,12 +31,12 @@ { "packager": "arduino", "name": "zephyr-sketch-tool", - "version": "0.4.0" + "version": "0.4.1" }, { "packager": "arduino", "name": "sync-zephyr-artifacts", - "version": "0.2.1" + "version": "0.2.2" }, { "packager": "arduino", @@ -67,6 +67,16 @@ "packager": "arduino", "name": "bin2uf2", "version": "0.1.0" + }, + { + "packager": "arduino", + "name": "scripting-tools", + "version": "0.0.7" + }, + { + "packager": "arduino", + "name": "rp2040tools", + "version": "1.1.0-rc1" } ] } diff --git a/extra/artifacts/zephyr_contrib.json b/extra/artifacts/zephyr_contrib.json index b2fa095dd..58fb1bc85 100644 --- a/extra/artifacts/zephyr_contrib.json +++ b/extra/artifacts/zephyr_contrib.json @@ -1,5 +1,5 @@ { - "name": "Zephyr Community Boards (BETA)", + "name": "Zephyr Community Boards", "architecture": "zephyr_contrib", "category": "Contributed" } diff --git a/extra/artifacts/zephyr_main.json b/extra/artifacts/zephyr_main.json index b36cef34b..6c15f125c 100644 --- a/extra/artifacts/zephyr_main.json +++ b/extra/artifacts/zephyr_main.json @@ -1,5 +1,5 @@ { - "name": "Arduino Zephyr Boards (BETA)", + "name": "Arduino Zephyr Boards", "architecture": "zephyr_main", "category": "Arduino" } diff --git a/extra/artifacts/zephyr_unoq.json b/extra/artifacts/zephyr_unoq.json index 53a3537c5..b0f6ba664 100644 --- a/extra/artifacts/zephyr_unoq.json +++ b/extra/artifacts/zephyr_unoq.json @@ -1,9 +1,9 @@ { - "name": "Arduino UNO Q Board", + "name": "Arduino Q Boards", "architecture": "zephyr", "category": "Arduino", "libraryDependencies": [ - { "name":"Arduino_RouterBridge", "version":"0.4.2" }, + { "name":"Arduino_RouterBridge", "version":"0.4.3" }, { "name":"Arduino_RPClite", "version":"0.3.0" }, { "name":"MsgPack", "version":"0.4.2" }, { "name":"DebugLog", "version":"0.8.4" }, diff --git a/extra/bootstrap.sh b/extra/bootstrap.sh index ef3d48929..f9ccde109 100755 --- a/extra/bootstrap.sh +++ b/extra/bootstrap.sh @@ -4,9 +4,10 @@ # SPDX-License-Identifier: Apache-2.0 set -e +set -o pipefail log_msg() { - if [ -n $GITHUB_WORKSPACE ] ; then + if [ -n "$GITHUB_WORKSPACE" ] ; then echo "::$1::$2" else echo " - $2" @@ -18,7 +19,34 @@ if [ ! -f platform.txt ]; then exit 2 fi -NEEDED_HALS=$(grep 'build.zephyr_hals=' boards.txt | cut -d '=' -f 2 | xargs -n 1 echo | sort -u | xargs echo) +REQUIRED_TOOLS="cmake curl git jq ninja pip3 python3 wget zip" +MISSING_TOOLS="" +for tool in $REQUIRED_TOOLS; do + if ! command -v "$tool" >/dev/null 2>&1; then + MISSING_TOOLS="$MISSING_TOOLS $tool" + fi +done +if [ -n "$MISSING_TOOLS" ]; then + echo "Missing required tools:$MISSING_TOOLS" >&2 + echo "Install them before running this script." >&2 + exit 2 +fi + +if [ -n "$WEST_CACHE" ]; then + # Set in CI context, uses a shared cache for west blobs and modules + mkdir -p $WEST_CACHE/blobs $WEST_CACHE/modules + WEST_BLOBS_CACHE="--cache-dirs $WEST_CACHE/blobs --auto-cache $WEST_CACHE/blobs" + WEST_MODULES_CACHE="--auto-cache $WEST_CACHE/modules" +fi + +get_unique_field_values() { + local field="$1" + local file="$2" + grep "$field=" $file | cut -d '=' -f 2 | xargs -n 1 echo | sort -u | xargs echo +} + +NEEDED_HALS=$(get_unique_field_values 'build.zephyr_hals' boards.txt) +NEEDED_TOOLCHAINS=$(get_unique_field_values 'build.zephyr_toolchain' boards.txt) HAL_FILTER="-hal_.*" for hal in $NEEDED_HALS; do @@ -28,21 +56,39 @@ done log_msg "group" "Bootstrapping Python environment for Zephyr" python3 -m venv venv source venv/bin/activate -pip install west protobuf grpcio-tools +pip3 install west protobuf grpcio-tools log_msg "endgroup" -log_msg "group" "Initializing Zephyr workspace and modules: $HAL_FILTER" -west init -l . +if ! [ -d ../.west ] ; then + log_msg "group" "Initializing Zephyr workspace and modules: $HAL_FILTER" + west init -l . +else + log_msg "warning" "Zephyr workspace already initialized, skipping west init" + log_msg "group" "Refreshing workspace and modules: $HAL_FILTER" +fi west config manifest.project-filter -- "$HAL_FILTER" -west update "$@" +west update $WEST_MODULES_CACHE "$@" west zephyr-export -pip install -r ../zephyr/scripts/requirements-base.txt +pip3 install -r ../zephyr/scripts/requirements-base.txt log_msg "endgroup" -log_msg "group" "Installing Zephyr SDK 0.16.8" -west sdk install --version 0.16.8 -t arm-zephyr-eabi -log_msg "endgroup" +TOOLCHAIN_VERSIONS=$(for tc in $NEEDED_TOOLCHAINS; do + version=$(jq -r --arg name "$tc" '.toolsDependencies[] | select(.name == $name) | .version' extra/artifacts/_common.json) + if [ -z "$version" ]; then + echo "No version found for toolchain '$tc' in extra/artifacts/_common.json" >&2 + exit 1 + fi + echo "$version $tc" +done | sort -V) + +for version in $(echo "$TOOLCHAIN_VERSIONS" | cut -d ' ' -f 1 | sort -u -V); do + toolchains=$(echo "$TOOLCHAIN_VERSIONS" | awk -v v="$version" '$1 == v { print $2 }') + log_msg "group" "Installing Zephyr SDK ${version}: ${toolchains}" + west sdk install --version "$version" -t $toolchains + log_msg "endgroup" +done +NEEDED_HALS="arduino-api $NEEDED_HALS" log_msg "group" "Fetching blobs for: $NEEDED_HALS" -west blobs fetch $NEEDED_HALS +west blobs $WEST_BLOBS_CACHE fetch $NEEDED_HALS log_msg "endgroup" diff --git a/extra/build.sh b/extra/build.sh index 72396c924..43c3b54e0 100755 --- a/extra/build.sh +++ b/extra/build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) Arduino s.r.l. and/or its affiliated companies # SPDX-License-Identifier: Apache-2.0 @@ -47,18 +47,22 @@ if ! [ -z "$chosen_board" ]; then # found, use the target and args from there board=$(jq -cr '.board' <<< "$chosen_board") target=$(jq -cr '.target' <<< "$chosen_board") - args=$(jq -cr '.args' <<< "$chosen_board") + # the args field is a single string; split it on unquoted whitespace, + # keeping quotes so a quoted value with spaces stays one array element + arg_token='(?:[^\s"'\'']+|"[^"]*"|'\''[^'\'']*'\'')+' + mapfile -t args < <(jq -cr '.args' <<< "$chosen_board" | grep -oP "$arg_token") upload_offset=$(jq -cr '.upload_offset' <<< "$chosen_board") # Check for debug flag and append if [ x$2 == x"--debug" ]; then - args="$args -- -DEXTRA_CONF_FILE=../extra/debug.conf" + args+=(-- -DEXTRA_CONF_FILE=../extra/debug.conf) fi else # expect Zephyr-compatible target and args target=$1 shift - args="$*" + # keep each argument as a separate array element to preserve quoting + args=("$@") chosen_board=$(extra/get_board_details.sh | jq -cr ".[] | select(.target == \"$target\") // empty") if [ ! -z "$chosen_board" ]; then board=$(jq -cr '.board' <<< "$chosen_board") @@ -68,8 +72,12 @@ else fi fi +# Save the build version to loader/VERSION and for later use in local files +build_version=$(extra/get_core_version.sh loader/VERSION) + echo -echo "Build target: $target $args" +echo "Build version: $build_version" +echo "Build target: $target ${args[*]}" # Get the variant name (NORMALIZED_BOARD_TARGET in Zephyr) variant=$(extra/get_variant_name.sh $target) @@ -85,7 +93,7 @@ fi BUILD_DIR=build/${variant} VARIANT_DIR=variants/${variant} rm -rf ${BUILD_DIR} -west build -d ${BUILD_DIR} -b ${target} loader -t llext-edk ${args} +west build -d ${BUILD_DIR} -b ${target} loader -t llext-edk "${args[@]}" # Extract the generated EDK tarball and copy it to the variant directory mkdir -p ${VARIANT_DIR} firmwares @@ -138,40 +146,54 @@ update_local_field() { local field=$1 local value="$2" local comment="$3" - local full_field_name="${board}.${field}" + + if [ -z "$board" ]; then + local file="platform.local.txt" + local full_field_name="${field}" + else + local file="boards.local.txt" + local full_field_name="${board}.${field}" + fi local match_regexp="${full_field_name//./\\.}" # escape dots + if [ ! -f $file ] ; then + cat << EOF > $file +######################################################################################### +# +# AUTO GENERATED FILE - DO NOT EDIT +# This file is manipulated by extra/build.sh; manual changes may be overwritten. +# +######################################################################################### + +EOF + fi + if [ -n "$comment" ]; then # if there's a comment, add/update it as a commented-out line above the actual field - if grep -qE "^# ${match_regexp}:" boards.local.txt; then - sed -i -e "s/^# ${match_regexp}:.*/# ${full_field_name}: ${comment}/" boards.local.txt + if grep -qE "^# ${match_regexp}:" $file; then + sed -i -e "s/^# ${match_regexp}:.*/# ${full_field_name}: ${comment}/" $file else - echo "# ${full_field_name}: ${comment}" >> boards.local.txt + echo "# ${full_field_name}: ${comment}" >> $file fi fi # update the actual field line - if grep -qE "^${match_regexp}" boards.local.txt; then - sed -i -e "s/^${match_regexp}=.*/${full_field_name}=${value}/" boards.local.txt + if grep -qE "^${match_regexp}" $file; then + sed -i -e "s/^${match_regexp}=.*/${full_field_name}=${value}/" $file else - echo "${full_field_name}=${value}" >> boards.local.txt + echo "${full_field_name}=${value}" >> $file fi } # update properties on boards.local.txt from the generated files if [ ! -z "$board" ]; then - if [ ! -f boards.local.txt ] ; then - cat << EOF > boards.local.txt -######################################################################################### -# -# AUTO GENERATED FILE - DO NOT EDIT -# This file is manipulated by extra/build.sh; manual changes may be overwritten. -# -######################################################################################### - -EOF - fi + # save version to both platform.local.txt and boards.local.txt: + # - the platform one is reported by the IDE as _the_ core version; + # - the board-specific one is used by the auto-update-loader feature + # (when developing, each board build should be tracked separately). + board="" update_local_field "version" "$build_version" + update_local_field "version" "$build_version" # sketch load address: start of sketch partition, hex (exact) CODE_ADDR=$(get_value_from_text_file variants/${variant}/syms-static.ld '_sketch_start') diff --git a/extra/build_all.sh b/extra/build_all.sh index 38c4174e1..70ddf5c1f 100755 --- a/extra/build_all.sh +++ b/extra/build_all.sh @@ -27,6 +27,9 @@ if [ ! -z "$GITHUB_STEP_SUMMARY" ] ; then echo "### Variant build results:" >> "$GITHUB_STEP_SUMMARY" fi +# This will force all boards to share the same build version +export PINNED_CORE_VERSION=$(extra/get_core_version.sh) + final_result=0 while read -r item; do board=$(jq -cr '.board' <<< "$item") diff --git a/extra/ci_calc_size_trends.py b/extra/ci_calc_size_trends.py new file mode 100755 index 000000000..d2f1edc6a --- /dev/null +++ b/extra/ci_calc_size_trends.py @@ -0,0 +1,922 @@ +#!/usr/bin/env python3 + +# Copyright (c) Arduino s.r.l. and/or its affiliated companies +# SPDX-License-Identifier: Apache-2.0 + +# Script to extract trends and flag outliers from the delta reports produced +# by extra/ci_calc_size_reports.py. +# +# This script expects a directory of "--link_mode=.json" +# report files (each already annotated with per-sketch "delta" data) and +# prints/writes a JSON summary of: +# - per-board/link_mode delta distributions (boards are not comparable to +# one another, so all statistics are computed within each board's own +# group of sketches, never across the whole dataset) +# - sketches whose delta is a statistical outlier vs. its own board's +# distribution +# - cross-board and static/dynamic link_mode trends + +import argparse +from collections import Counter, defaultdict +import html +import io +import json +from pathlib import Path +import re +import statistics + +SIZE_NAMES = ("flash", "RAM for global variables") + +# deltas smaller than this (in bytes) are treated as noise: common +# per-binary overhead changes that would otherwise dominate a report +# without being interesting +MIN_DELTA = 128 + +def parse_filename(name): + """ + Parse a report filename of the form + "---link_mode=.json" into its + (package, board, link_mode) components, matching the naming convention + used by ci_calc_size_reports.py. + """ + + package, board, opts = name[:-5].split('-')[1:4] + link_mode = opts.split('=', 1)[1] if '=' in opts else opts + return package, board, link_mode + +def load_records(input_dir): + """ + Load every report JSON file in input_dir and flatten it into one record + per successfully-compiled sketch, keeping only sketches with valid + (numeric) flash and RAM deltas. + """ + + records = [] + for path in sorted(Path(input_dir).glob("*.json")): + with open(path) as f: + data = json.load(f) + package, board, link_mode = parse_filename(path.name) + for sketch in data["boards"][0]["sketches"]: + if not sketch.get("compilation_success", False): + continue + sizes = {entry["name"]: entry for entry in sketch["sizes"]} + flash = sizes.get("flash") + ram = sizes.get("RAM for global variables") + if not flash or not ram: + continue + flash_delta = flash.get("delta") + ram_delta = ram.get("delta") + if not flash_delta or not ram_delta: + continue + if isinstance(flash_delta["absolute"], str) or isinstance(ram_delta["absolute"], str): + # "N/A", previous size was zero/missing + continue + records.append({ + "package": package, + "board": board, + "link_mode": link_mode, + "sketch": sketch["name"], + "flash_delta_abs": flash_delta["absolute"], + "flash_delta_rel": flash_delta["relative"], + "ram_delta_abs": ram_delta["absolute"], + "ram_delta_rel": ram_delta["relative"], + "flash_max": flash["maximum"], + "ram_max": ram["maximum"], + }) + return records + +def mad_outliers(records, key, threshold=3.5): + """ + Flag records whose field is a robust statistical outlier, using the + median absolute deviation (MAD) modified z-score computed WITHIN each + board/link_mode's own group of sketches. + """ + + by_board = defaultdict(list) + for record in records: + by_board[(record["board"], record["link_mode"])].append(record) + + outliers = [] + for group in by_board.values(): + values = [r[key] for r in group] + median = statistics.median(values) + mad = statistics.median(abs(v - median) for v in values) + for record in group: + value = record[key] + if mad == 0: + # every non-outlier record in this board shares the exact + # same value: anything else stands out regardless of score + is_outlier = value != median + score = None + else: + score = 0.6745 * (value - median) / mad + is_outlier = abs(score) > threshold + if is_outlier: + outliers.append({**record, "board_median": median, "mad_score": score}) + return outliers + +def per_board_summary(records): + """ + Summarize flash/RAM deltas per (package, board, link_mode) group. + """ + + by_board = defaultdict(list) + for record in records: + by_board[(record["package"], record["board"], record["link_mode"])].append(record) + + summary = [] + for (package, board, link_mode), group in sorted(by_board.items()): + flash_vals = [r["flash_delta_abs"] for r in group] + ram_vals = [r["ram_delta_abs"] for r in group] + summary.append({ + "package": package, + "board": board, + "link_mode": link_mode, + "sketch_count": len(group), + "flash_delta_min": min(flash_vals), + "flash_delta_mean": statistics.mean(flash_vals), + "flash_delta_median": statistics.median(flash_vals), + "flash_delta_max": max(flash_vals), + "ram_delta_min": min(ram_vals), + "ram_delta_mean": statistics.mean(ram_vals), + "ram_delta_median": statistics.median(ram_vals), + "ram_delta_max": max(ram_vals), + }) + return summary + +def outlier_sketch_trends(records, flash_outliers, ram_outliers): + """ + For every sketch that was flagged as a flash or RAM outlier on at least + one board, list its delta on every board/link_mode it appears on. This + shows whether a flagged anomaly is isolated to one board or a wider + cross-board trend, without dumping every sketch's minor per-board + variation (most of which is not an outlier, just normal board-specific + noise). + """ + + flagged_sketches = {o["sketch"] for o in flash_outliers} | {o["sketch"] for o in ram_outliers} + if not flagged_sketches: + return [] + + by_sketch = defaultdict(list) + for record in records: + if record["sketch"] in flagged_sketches: + by_sketch[record["sketch"]].append(record) + + trends = [] + for sketch, group in by_sketch.items(): + entries = sorted( + ({"board": r["board"], "link_mode": r["link_mode"], + "flash_delta_abs": r["flash_delta_abs"], "ram_delta_abs": r["ram_delta_abs"]} + for r in group), + key=lambda e: (e["board"], e["link_mode"]), + ) + trends.append({ + "sketch": sketch, + "entries": entries, + "max_abs_delta": max(abs(e["flash_delta_abs"]) + abs(e["ram_delta_abs"]) for e in entries), + }) + + trends.sort(key=lambda t: -t["max_abs_delta"]) + return trends + +def _read_commit_info(input_dir): + """ + Read the (head_sha, head_url) the reports were generated for, from the + first report JSON file found. All files in one CI run describe the + same commit, so any one file's values are representative. + """ + + for path in sorted(Path(input_dir).glob("*.json")): + with open(path) as f: + data = json.load(f) + return data.get("commit_hash"), data.get("commit_url") + return None, None + +def build_trends(input_dir): + """ + Load all report records from input_dir and compute the full trends + summary: per-board outliers, per-board summary, and cross-board outlier + trends. + """ + + records = load_records(input_dir) + head_sha, head_url = _read_commit_info(input_dir) + flash_outliers = mad_outliers(records, "flash_delta_abs") + ram_outliers = mad_outliers(records, "ram_delta_abs") + return { + "head_sha": head_sha, + "head_url": head_url, + "record_count": len(records), + "board_summary": per_board_summary(records), + "flash_outliers": flash_outliers, + "ram_outliers": ram_outliers, + "outlier_sketch_trends": outlier_sketch_trends(records, flash_outliers, ram_outliers), + }, records + +def _truncate(value): + """Truncate a byte-size value to an integer for display.""" + + return int(value) + +def _base_url(head_url, base_sha): + """ + Derive the base commit's URL from the head commit's URL (same repo, + GitHub's .../commit/ pattern), so the base is also a link + whenever we have enough information to build one. + """ + + if not head_url or not base_sha: + return None + return head_url.rsplit("/", 1)[0] + "/" + base_sha + +def _compare_url(head_url, base_sha, head_sha): + """ + Build a GitHub compare view URL between base_sha and head_sha, derived + from the head commit's URL the same way _base_url derives the base + commit's own link. + """ + + if not head_url or not base_sha or not head_sha: + return None + repo_url = head_url.rsplit("/", 2)[0] + return f"{repo_url}/compare/{base_sha}...{head_sha}" + +def _outlier_rows(outliers, top_n): + """Rank outlier records by combined magnitude and return the top N as plain tuples.""" + + ranked = sorted(outliers, key=lambda o: -(abs(o["flash_delta_abs"]) + abs(o["ram_delta_abs"]))) + return [ + (o["board"], o["link_mode"], o["sketch"], + _truncate(o["flash_delta_abs"]), _truncate(o["ram_delta_abs"])) + for o in ranked[:top_n] + ] + +def _build_report_data(trends, top_n, base_sha=None): + """ + Assemble every prepared row set needed for a report, as plain Python + values with no markup of any kind. Keeping this decoupled from any + particular output format means a renderer for a different format (e.g. + Markdown) could reuse the exact same rows instead of re-deriving them. + """ + + board_rows = [ + (b["package"], b["board"], b["link_mode"], b["sketch_count"], + _truncate(b["flash_delta_min"]), _truncate(b["flash_delta_mean"]), + _truncate(b["flash_delta_median"]), _truncate(b["flash_delta_max"]), + _truncate(b["ram_delta_min"]), _truncate(b["ram_delta_mean"]), + _truncate(b["ram_delta_median"]), _truncate(b["ram_delta_max"])) + for b in trends["board_summary"] + ] + + trend_rows = [ + (t["sketch"], e["board"], e["link_mode"], + _truncate(e["flash_delta_abs"]), _truncate(e["ram_delta_abs"])) + for t in trends["outlier_sketch_trends"][:top_n] + for e in t["entries"] + if abs(e["flash_delta_abs"]) >= MIN_DELTA or abs(e["ram_delta_abs"]) >= MIN_DELTA + ] + + return { + "head_sha": trends["head_sha"], + "head_url": trends["head_url"], + "base_sha": base_sha, + "base_url": _base_url(trends["head_url"], base_sha), + "compare_url": _compare_url(trends["head_url"], base_sha, trends["head_sha"]), + "record_count": trends["record_count"], + "n_board_groups": len(trends["board_summary"]), + "n_flash_outliers": len(trends["flash_outliers"]), + "n_ram_outliers": len(trends["ram_outliers"]), + "top_n": top_n, + "board_rows": board_rows, + "flash_outlier_rows": _outlier_rows(trends["flash_outliers"], top_n), + "ram_outlier_rows": _outlier_rows(trends["ram_outliers"], top_n), + "trend_rows": trend_rows, + } + +class _Safe(str): + """A table cell that is already-safe HTML and must not be escaped again.""" + +class _Cell: + """A table cell with an optional inline style (e.g. a background color).""" + + def __init__(self, text, style=None): + self.text = text + self.style = style + +# same hue as the chart backgrounds added later (pale green below zero, pale +# red above); gray marks a value that falls below the MIN_DELTA threshold +NEGATIVE_CELL_STYLE = "background-color: rgba(46, 160, 44, 0.35);" +POSITIVE_CELL_STYLE = "background-color: rgba(214, 39, 40, 0.35);" +NEUTRAL_CELL_STYLE = "background-color: rgba(136, 136, 136, 0.3);" + +def _mono(text): + """Render a package/board identifier in a typewriter (monospace) font.""" + + return _Safe(f"{html.escape(str(text))}") + +def _board_name(text): + """Render a board name bold and in a typewriter (monospace) font.""" + + return _Safe(f"{html.escape(str(text))}") + +def _bold_board_label(label): + """Bold+monospace just the board name portion of a "board (link_mode)" label.""" + + match = re.match(r"^(.*) \((\w+)\)$", label) + if not match: + return html.escape(label) + board, link_mode = match.groups() + return f"{html.escape(board)} ({html.escape(link_mode)})" + +def _delta_cell(value, text=None): + """ + A delta-value cell shaded green/red by sign, or gray if the value falls + below the MIN_DELTA threshold (including exactly zero). Positive values + are shown with an explicit "+" sign, matching the Markdown report. + """ + + if abs(value) < MIN_DELTA: + style = NEUTRAL_CELL_STYLE + elif value < 0: + style = NEGATIVE_CELL_STYLE + else: + style = POSITIVE_CELL_STYLE + if text is None: + text = f"{value:+d}" if value != 0 else "0" + return _Cell(text, style) + +def _render_td(cell): + if isinstance(cell, _Cell): + text = str(cell.text) if isinstance(cell.text, _Safe) else html.escape(str(cell.text)) + style = f' style="{cell.style}"' if cell.style else "" + return f"{text}" + text = str(cell) if isinstance(cell, _Safe) else html.escape(str(cell)) + return f"{text}" + +_NO_ROWS_MESSAGE = "

none

" + +def _commit_html(sha, url): + """Render a commit as a short linked hash, or "unknown" if absent.""" + + if not sha: + return "unknown" + short = html.escape(sha[:10]) + return f'{short}' if url else f"{short}" + +def _html_table(headers, rows): + if not rows: + return _NO_ROWS_MESSAGE + head = "".join(f"{html.escape(str(h))}" for h in headers) + body = "".join( + "" + "".join(_render_td(cell) for cell in row) + "" + for row in rows + ) + return f"{head}{body}
" + +def _html_table_grouped(solo_headers, group_headers, sub_headers, rows): + """ + A table with a two-row header: solo_headers span both header rows + (e.g. "board"), and each name in group_headers spans len(sub_headers) + columns in the top row, with sub_headers repeated underneath it (e.g. + "Flash" / "RAM", each split into "min" / "mean" / "median" / "max"). + """ + + if not rows: + return _NO_ROWS_MESSAGE + + row1 = "".join(f'{html.escape(h)}' for h in solo_headers) + row1 += "".join(f'{html.escape(g)}' for g in group_headers) + row2 = "".join(f"{html.escape(h)}" for h in sub_headers * len(group_headers)) + + body = "".join( + "" + "".join(_render_td(cell) for cell in row) + "" + for row in rows + ) + return f"{row1}{row2}{body}
" + +def _figure_to_inline_svg(fig, plt, bold_mono_labels=(), tooltips=None): + """ + Render a matplotlib figure to inline SVG markup (real // + elements, not baked-in glyph outlines or a raster image), so it + can be embedded directly in an HTML report with no external files and + no JavaScript. + + bold_mono_labels: tick-label strings of the form "name (mode)" whose + "name" portion should render bold+monospace while " (mode)" stays in + the chart's normal font β€” matching the board/link_mode split styling + used elsewhere in the HTML tables. matplotlib text has no per-run + styling, and SVG has no inline HTML either, but SVG's own equivalent + (, mixing styles within one element) does the same job, + so the label's exact text content is swapped for a -wrapped + version after rendering. + + tooltips: {gid: text} for artists whose set_gid(gid) was called before + rendering (e.g. one scatter() call per point). matplotlib wraps each + such artist in ...; a inserted as its first + child gives it a native hover tooltip, no JavaScript needed, same as + the "title" attribute would in plain HTML. + """ + + buf = io.BytesIO() + fig.savefig(buf, format="svg", bbox_inches="tight") + plt.close(fig) + svg = buf.getvalue().decode("utf-8") + svg = svg[svg.index("<svg"):] + + # matplotlib resolves "system-ui"/"monospace" to a long quoted fallback + # chain for its own local rendering and writes that verbatim into the + # SVG's CSS; browsers treat a *quoted* 'system-ui' as a literal (and + # nonexistent) font name rather than the special unquoted keyword, so + # rewrite both chains down to the plain generic keywords that actually + # match the surrounding HTML page's font instead of matplotlib's + # locally-installed fallback fonts. + svg = re.sub(r"font-family: [^;]*sans-serif;", "font-family: system-ui, sans-serif;", svg) + svg = re.sub(r"font-family: [^;]*monospace;", "font-family: monospace;", svg) + + for label in bold_mono_labels: + match = re.match(r"^(.*) \((\w+)\)$", label) + if not match: + continue + board, mode = match.groups() + old = f">{html.escape(label)}<" + new = (f'><tspan style="font-weight:bold;font-family:monospace">' + f'{html.escape(board)}</tspan> ({html.escape(mode)})<') + svg = svg.replace(old, new) + + for gid, text in (tooltips or {}).items(): + svg = re.sub( + rf'(<g id="{re.escape(gid)}"[^>]*>)', + rf'\1<title>{html.escape(text)}', + svg, count=1, + ) + + return _Safe(svg) + +def _chart_bubble(records, key, label, plt, min_abs_delta=MIN_DELTA): + """ + Bubble chart of one metric's delta distribution, one row per + board/link_mode that has at least one delta reaching min_abs_delta, + ordered by that board's largest delta (descending) so the most notable + boards group at the top. Each bubble's x position is a delta value + actually seen on that board, and its size encodes how many sketches on + that board share that exact value. Boards with no delta reaching the + threshold are left out of the plot entirely; instead, a text summary + grouping those boards by their typical (median) delta is returned + alongside the chart, so the report is not padded with empty rows. + + Returns (inline_svg_markup_or_None, excluded_rows). + """ + + all_values = defaultdict(list) + qualifying = defaultdict(list) + for record in records: + board_label = f"{record['board']} ({record['link_mode']})" + value = record[key] + all_values[board_label].append(value) + if abs(value) >= min_abs_delta: + qualifying[board_label].append(value) + + plotted_boards = sorted( + qualifying.keys(), + key=lambda l: (-max(abs(v) for v in qualifying[l]), l), + ) + excluded_boards = sorted(set(all_values) - set(qualifying)) + + img = None + if plotted_boards: + xs, ys, counts = [], [], [] + for row, board_label in enumerate(plotted_boards): + for value, count in Counter(qualifying[board_label]).items(): + xs.append(value) + ys.append(row) + counts.append(count) + max_count = max(counts) + sizes = [30 + 220 * (count / max_count) for count in counts] + colors = ["#888888" if v == 0 else "#217821" if v < 0 else "#a31f1f" for v in xs] + + fig, ax = plt.subplots(figsize=(9, max(3, 0.4 * len(plotted_boards)))) + # one scatter() call per point (instead of one batched call for all + # of them) so each bubble gets its own gid, letting the SVG + # post-processing below give it its own hover tooltip + tooltips = {} + for i, (x_val, y_val, size, color, count, board_label) in enumerate( + zip(xs, ys, sizes, colors, counts, (plotted_boards[y] for y in ys)) + ): + gid = f"bubble-{label}-{i}" + ax.scatter([x_val], [y_val], s=[size], color=color, alpha=0.85, + edgecolors="#333333", linewidths=0.5, zorder=3, gid=gid) + sketch_word = "sketch" if count == 1 else "sketches" + tooltips[gid] = f"{board_label}: {x_val:+d} bytes ({count} {sketch_word})" + for x_val, y_val in zip(xs, ys): + ax.annotate(f"{x_val:+d}", (x_val, y_val), xytext=(0, -8), textcoords="offset points", + ha="center", va="top", fontsize=6, color="#333333", zorder=4) + + # mean and 1-sigma window over EVERY sketch on that board (not just + # the ones clearing min_abs_delta), matching the population that + # per_board_summary()/mad_outliers() already summarize elsewhere + row_stats = {} + for row, board_label in enumerate(plotted_boards): + values = all_values[board_label] + mean = statistics.mean(values) + stdev = statistics.stdev(values) if len(values) > 1 else 0.0 + row_stats[row] = (mean, stdev) + + for row, (mean, stdev) in row_stats.items(): + ax.fill_betweenx([row - 0.4, row + 0.4], mean - stdev, mean + stdev, + color="#add8e6", alpha=0.4, zorder=1, + label="mean Β± Οƒ" if row == 0 else None) + ax.plot([mean, mean], [row - 0.4, row + 0.4], linewidth=1.5, + color="#1f4e79", zorder=2, + label="mean" if row == 0 else None) + ax.legend(loc="lower right", fontsize=7, frameon=False) + + ax.set_yticks(range(len(plotted_boards))) + ax.set_yticklabels(plotted_boards) + ax.set_ylim(len(plotted_boards) - 0.5, -0.5) + + # use the actual data extent, not ax.get_xlim(): matplotlib's linear + # autoscale already adds its own default margin (~5% of the *total* + # span) to both sides, which on lopsided data inflates the small + # side's bound far more (in ratio terms) than the large side's + data_min = min(xs + [0]) + data_max = max(xs + [0]) + # multiplicative pad, not additive: on a log-ish axis, equal visual + # margin means equal *ratio*, so both sides are scaled by the same + # factor regardless of how differently sized their magnitudes are + pad_ratio = 1.3 + left = data_min * pad_ratio if data_min < 0 else -min_abs_delta * pad_ratio + right = data_max * pad_ratio if data_max > 0 else min_abs_delta * pad_ratio + xlim = (left, right) + ax.axvspan(xlim[0], -min_abs_delta, color="#2ca02c", alpha=0.18, zorder=0) + ax.axvspan(min_abs_delta, xlim[1], color="#d62728", alpha=0.18, zorder=0) + ax.axvspan(-min_abs_delta, min_abs_delta, color="#888888", alpha=0.25, zorder=0.5) + ax.set_xlim(xlim) + # symlog, not log: deltas can be negative or zero, and every plotted + # point already clears min_abs_delta, so the linear region just + # covers the (empty) noise band without needing to render it + ax.set_xscale("symlog", linthresh=min_abs_delta, linscale=0.3) + # "10k"/"1M" instead of matplotlib's default "10^4"/"10^6" tick labels + from matplotlib.ticker import EngFormatter + ax.xaxis.set_major_formatter(EngFormatter(sep="")) + + ax.axvline(0, color="#999", linewidth=0.8, linestyle="--", zorder=1) + ax.set_xlabel(f"{label} delta (bytes, |delta| >= {min_abs_delta})") + ax.set_title(f"{label} delta distribution per board / link_mode") + fig.tight_layout() + img = _figure_to_inline_svg(fig, plt, bold_mono_labels=plotted_boards, tooltips=tooltips) + + by_value = defaultdict(list) + for board_label in excluded_boards: + median_value = _truncate(statistics.median(all_values[board_label])) + by_value[median_value].append(board_label) + + def value_label(value): + return "unchanged" if value == 0 else f"{value:+d}" + + excluded_rows = [ + (_delta_cell(value, text=value_label(value)), _Safe(", ".join(_bold_board_label(b) for b in boards))) + for value, boards in sorted(by_value.items()) + ] + + return img, excluded_rows + +def _accordion(group_name, items): + """ + Render (label, table_html) pairs as a set of
+ elements. Sharing one `name` across a group of
is native + HTML: the browser keeps at most one of them open at a time within that + name, with no JavaScript required. + """ + + def render_label(label): + return str(label) if isinstance(label, _Safe) else html.escape(str(label)) + + return "\n\n".join( + f'
\n{render_label(label)}\n\n{table_html}\n\n
' + for label, table_html in items + ) + +def generate_html(report_data, records): + """ + Render prepared report data as a single self-contained HTML page: charts + are rendered with matplotlib as inline SVG, so the output file has no + external dependencies, no JavaScript, and can be opened/shared as-is. + """ + + import logging + logging.getLogger("matplotlib.font_manager").setLevel(logging.ERROR) + + import matplotlib + matplotlib.use("Agg") + matplotlib.rcParams["svg.fonttype"] = "none" + matplotlib.rcParams["font.family"] = "system-ui, sans-serif" + import matplotlib.pyplot as plt + + flash_box_img, flash_box_excluded = _chart_bubble(records, "flash_delta_abs", "Flash", plt) + ram_box_img, ram_box_excluded = _chart_bubble(records, "ram_delta_abs", "RAM", plt) + + head_html = _commit_html(report_data["head_sha"], report_data["head_url"]) + base_html = _commit_html(report_data["base_sha"], report_data["base_url"]) + compare_html = f' (compare)' if report_data["compare_url"] else "" + + board_rows = [ + (_mono(row[0]), _board_name(row[1]), row[2], row[3], *(_delta_cell(v) for v in row[4:])) + for row in report_data["board_rows"] + ] + flash_outlier_rows = [ + (_board_name(board), link_mode, sketch, _delta_cell(flash), _delta_cell(ram)) + for board, link_mode, sketch, flash, ram in report_data["flash_outlier_rows"] + ] + ram_outlier_rows = [ + (_board_name(board), link_mode, sketch, _delta_cell(flash), _delta_cell(ram)) + for board, link_mode, sketch, flash, ram in report_data["ram_outlier_rows"] + ] + trend_rows = [ + (sketch, _board_name(board), link_mode, _delta_cell(flash), _delta_cell(ram)) + for sketch, board, link_mode, flash, ram in report_data["trend_rows"] + ] + + top_n = report_data["top_n"] + + def top_suffix(total): + return f" (top {top_n} of {total})" if top_n < total else "" + + raw_by_board = defaultdict(list) + raw_by_sketch = defaultdict(list) + for record in records: + raw_by_board[(record["package"], record["board"], record["link_mode"])].append(record) + raw_by_sketch[record["sketch"]].append(record) + + def notable_counts(recs): + """ + Count notable (not ignored, i.e. clears MIN_DELTA) flash/RAM values + in recs, split by sign: negative (green) vs. positive (red). + """ + + green = red = 0 + for r in recs: + for key in ("flash_delta_abs", "ram_delta_abs"): + value = r[key] + if value >= MIN_DELTA: + red += 1 + elif value <= -MIN_DELTA: + green += 1 + return green, red + + def _count_dot(count, color): + return (f'{count}') + + def notable_suffix(recs): + green, red = notable_counts(recs) + parts = [] + if green: + parts.append(_count_dot(green, "#217821")) + if red: + parts.append(_count_dot(red, "#a31f1f")) + return f' ({", ".join(parts)})' if parts else "" + + def board_table_items(): + for (package, board, link_mode), recs in sorted(raw_by_board.items()): + recs = sorted(recs, key=lambda r: r["sketch"]) + rows = [ + (_mono(r["sketch"]), _delta_cell(_truncate(r["flash_delta_abs"])), + _delta_cell(_truncate(r["ram_delta_abs"]))) + for r in recs + ] + label = _Safe( + f"{_mono(package)} {_board_name(board)} ({html.escape(link_mode)}) " + f"- {len(rows)} sketches{notable_suffix(recs)}" + ) + yield label, _html_table(["sketch", "flash Ξ”", "RAM Ξ”"], rows) + + def sketch_table_items(): + for sketch, recs in sorted(raw_by_sketch.items()): + recs = sorted(recs, key=lambda r: (r["board"], r["link_mode"])) + rows = [ + (_board_name(r["board"]), r["link_mode"], _delta_cell(_truncate(r["flash_delta_abs"])), + _delta_cell(_truncate(r["ram_delta_abs"]))) + for r in recs + ] + label = _Safe(f"{_mono(sketch)} - {len(rows)} boards{notable_suffix(recs)}") + yield label, _html_table(["board", "link_mode", "flash Ξ”", "RAM Ξ”"], rows) + + board_tables = _accordion("board-details", board_table_items()) + sketch_tables = _accordion("sketch-details", sketch_table_items()) + + return f""" + + + +Size delta trends report + + + +

Size deltas from {base_html} to {head_html}{compare_html}

+

{report_data["record_count"]} sketch/board/link_mode records across +{report_data["n_board_groups"]} board/link_mode combinations. +{report_data["n_flash_outliers"]} flash and {report_data["n_ram_outliers"]} RAM per-board outliers flagged. +

+ +

Per-board delta distributions

+{flash_box_img or ""} +

Boards below the {MIN_DELTA} byte threshold (not plotted above):

+{_html_table(["delta", "boards"], flash_box_excluded)} +{ram_box_img or ""} +

Boards below the {MIN_DELTA} byte threshold (not plotted above):

+{_html_table(["delta", "boards"], ram_box_excluded)} + +

Per-board / link_mode summary

+{_html_table_grouped(["package", "board", "link_mode", "sketches"], ["Flash", "RAM"], + ["min", "mean", "median", "max"], board_rows)} + +

Flash delta outliers{top_suffix(report_data["n_flash_outliers"])}

+{_html_table(["board", "link_mode", "sketch", "flash Ξ”", "RAM Ξ”"], flash_outlier_rows)} + +

RAM delta outliers{top_suffix(report_data["n_ram_outliers"])}

+{_html_table(["board", "link_mode", "sketch", "flash Ξ”", "RAM Ξ”"], ram_outlier_rows)} + +

Outlier sketches across boards

+

Full per-board pattern for every sketch flagged above, showing +whether the anomaly is isolated to one board or a wider trend.

+{_html_table(["sketch", "board", "link_mode", "flash Ξ”", "RAM Ξ”"], trend_rows)} + +

Per-board delta tables

+

Every sketch's flash/RAM delta on that board/link_mode. Opening one closes any other open here.

+{board_tables} + +

Per-sketch delta tables

+

Every board/link_mode a sketch was built for, and its flash/RAM delta there. Opening one closes any other open here.

+{sketch_tables} + + +""" + +def _md_code(text): + """Render a package/board identifier in a typewriter (monospace) font.""" + + return f"`{text}`" + +def _md_number(value): + """ + Bold a delta value if its magnitude clears the MIN_DELTA threshold, and + prefix it with a small colored dot by sign (green for negative, red for + positive). Values that don't clear the threshold are ignored deltas: + plain, unmarked numbers, so the dot only ever flags a genuinely notable + value. GitHub is the only Markdown renderer this targets, so nested + / is safe to rely on; it also shrinks and vertically centers + the circle so it renders as a small dot instead of a full-size emoji + that skews table row height. + """ + + text = f"{value:+d}" if value != 0 else "0" + if abs(value) <= MIN_DELTA: + return text + marker = "🟒" if value < 0 else "πŸ”΄" + return f"{marker} **{text}**" + +def _commit_md(sha, url): + """Render a commit as a short linked hash, or "unknown" if absent.""" + + if not sha: + return "unknown" + short = sha[:10] + return f"[`{short}`]({url})" if url else f"`{short}`" + +def _markdown_table(headers, rows, align=None): + """ + align is an optional per-column list of "l"/"r" (default all "l"); + "r" right-aligns a numeric column via the GFM `---:` separator syntax. + """ + + if not rows: + return "_none_" + if align is None: + align = ["l"] * len(headers) + sep_cell = {"l": "---", "r": "---:"} + lines = [ + "| " + " | ".join(str(h) for h in headers) + " |", + "| " + " | ".join(sep_cell[a] for a in align) + " |", + ] + lines += ["| " + " | ".join(str(cell) for cell in row) + " |" for row in rows] + return "\n".join(lines) + +def generate_markdown(report_data): + """ + Render prepared report data as a GitHub-flavored Markdown report: a + one-line count recap, the outlier-sketch cross-board trend table, the + per-board summary rows with at least one notable (>= MIN_DELTA) value, + and the full per-board summary folded into a collapsed section. + """ + + board_headers = ["package", "board", "link_mode", "sketches", + "flash min", "flash mean", "flash median", "flash max", + "ram min", "ram mean", "ram median", "ram max"] + board_align = ["l", "l", "l"] + ["r"] * (len(board_headers) - 3) + trend_align = ["l", "l", "l", "r", "r"] + + def format_board_row(row): + package, board, link_mode, sketch_count, *deltas = row + return (_md_code(package), _md_code(board), link_mode, sketch_count, + *(_md_number(v) for v in deltas)) + + raw_board_rows = report_data["board_rows"] + notable_raw_rows = [row for row in raw_board_rows if any(abs(v) >= MIN_DELTA for v in row[4:])] + notable_board_rows = [format_board_row(row) for row in notable_raw_rows] + + by_package = defaultdict(list) + for raw_row in raw_board_rows: + by_package[raw_row[0]].append(format_board_row(raw_row)[1:]) + + full_sections = "\n\n".join( + f"
\nFull per-board/link-mode summary for {_mono(package)}\n\n" + f"{_markdown_table(board_headers[1:], rows, board_align[1:])}\n\n
" + for package, rows in by_package.items() + ) + + trend_rows = [ + (sketch, _md_code(board), link_mode, _md_number(flash), _md_number(ram)) + for sketch, board, link_mode, flash, ram in report_data["trend_rows"] + ] + + def short_sha(sha): + return sha[:10] if sha else "unknown" + + head_sha = short_sha(report_data["head_sha"]) + base_sha = short_sha(report_data["base_sha"]) + compare_suffix = f' ([compare]({report_data["compare_url"]}))' if report_data["compare_url"] else "" + + recap = ( + f'{report_data["record_count"]} sketch/board/link_mode records across ' + f'{report_data["n_board_groups"]} board/link_mode combinations. ' + f'{report_data["n_flash_outliers"]} flash and {report_data["n_ram_outliers"]} RAM per-board ' + "outliers flagged." + ) + + return f""" +--- + +## Size deltas from {base_sha} to {head_sha}{compare_suffix} + +{recap} + +## Per-board / link_mode summary + +{_markdown_table(board_headers, notable_board_rows, board_align)} + +{full_sections} + +## Outlier sketches across boards + +{_markdown_table(["sketch", "board", "link_mode", "flash Ξ”", "RAM Ξ”"], trend_rows, trend_align)} +""" + +def main(): + parser = argparse.ArgumentParser( + description="Extract trends and flag outliers from ci_calc_size_reports.py delta reports") + parser.add_argument("--input-dir", default=".", + help="directory containing delta report JSON files (default: current directory)") + parser.add_argument("--output-json", + help="write the trends summary as JSON to this path (default: print to stdout " + "if --output-html is not given either)") + parser.add_argument("--output-html", + help="write a self-contained HTML report (charts + tables, no external assets) " + "to this path; requires matplotlib") + parser.add_argument("--output-md", + help="write a GitHub-flavored Markdown report (counts recap, outlier tables, " + "per-board summary) to this path") + parser.add_argument("--base-sha", + help="SHA of the baseline commit the deltas were computed against (the " + "reports themselves don't record it); shown in the HTML/Markdown " + "report header alongside the commit they were generated for") + args = parser.parse_args() + + trends, records = build_trends(args.input_dir) + + if args.output_json: + Path(args.output_json).write_text(json.dumps(trends, indent=2)) + print(f"Wrote trends JSON to {args.output_json}") + elif not args.output_html and not args.output_md: + print(json.dumps(trends, indent=2)) + + if args.output_html or args.output_md: + report_data = _build_report_data(trends, top_n=30, base_sha=args.base_sha) + if args.output_html: + Path(args.output_html).write_text(generate_html(report_data, records)) + print(f"Wrote trends HTML report to {args.output_html}") + if args.output_md: + Path(args.output_md).write_text(generate_markdown(report_data)) + print(f"Wrote trends Markdown report to {args.output_md}") + +if __name__ == "__main__": + main() diff --git a/extra/ci_collect_data.py b/extra/ci_collect_data.py index 9d52799fb..321eb287e 100755 --- a/extra/ci_collect_data.py +++ b/extra/ci_collect_data.py @@ -134,12 +134,22 @@ def log_test(artifact, board, sketch, link_mode, exceptions, status, issues, job # Get list of expected errors for this board exceptions = [] + warning_filters = [] # [ (sketch_pattern, regex), ...] if os.path.exists(f"variants/{variant}/known_example_issues.txt"): with open(f"variants/{variant}/known_example_issues.txt", 'r') as f: for line in f: - sketch_pattern = line.split('#')[0].strip() - if sketch_pattern: - exceptions.append(re.compile(f"^(ArduinoCore-zephyr/)?{sketch_pattern}")) + line = line.split('#')[0].strip() + if not line: + continue + elif '|' in line: + # this is a warning filter, not an exception + sketch_pattern, warning_pattern = line.split('|', 1) + sketch_pattern = re.compile(f"^(ArduinoCore-zephyr/)?{sketch_pattern}") + warning_pattern = re.compile(warning_pattern) + warning_filters.append(( sketch_pattern, warning_pattern )) + else: + # this is an exception + exceptions.append(re.compile(f"^(ArduinoCore-zephyr/)?{line}")) # Get raw data from report files for both static and dynamic linking modes for link_mode in ("static", "dynamic"): @@ -176,6 +186,11 @@ def log_test(artifact, board, sketch, link_mode, exceptions, status, issues, job # Replace long absolute paths with '...' for brevity. sketch_issues = [ re.sub(r'(/.+?)((/[^/]+){3}):', r'...\2:', issue) for issue in issues ] + # Remove known warnings, if any + for sketch_pattern, warning_pattern in warning_filters: + if sketch_pattern.match(sketch): + sketch_issues = [ issue for issue in sketch_issues if not warning_pattern.search(issue) ] + if not success: status = ERROR elif len(sketch_issues): # Implies warnings/non-critical issues diff --git a/extra/get_board_details.sh b/extra/get_board_details.sh index cbe1a1178..a70d8b8b0 100755 --- a/extra/get_board_details.sh +++ b/extra/get_board_details.sh @@ -6,13 +6,13 @@ set -e get_boards() { - cat boards.txt | sed -e 's/\s*#.*//' | grep -E '^.*\.build\.variant=' | sed -e 's/\.build\.variant=.*//' + cat boards.txt | sed -e 's/\s*#.*//' | grep -E '^.*\.build\.variant=' | sed -e 's/\.build\.variant=.*//' -e 's/["\\]/\\&/g' } get_board_field() { board=$1 field=$2 - cat boards.txt | sed -e 's/\s*#.*//' | grep -E "^$board\\.$field=" | cut -d '=' -f2- | sed -e 's/"/\"/g' + cat boards.txt | sed -e 's/\s*#.*//' | grep -E "^$board\\.$field=" | cut -d '=' -f2- | sed -e 's/["\\]/\\&/g' } for BOARD in $(get_boards); do diff --git a/extra/get_core_version.sh b/extra/get_core_version.sh index 51171cee6..a914a61a1 100755 --- a/extra/get_core_version.sh +++ b/extra/get_core_version.sh @@ -15,7 +15,7 @@ # # A bash regex extracts the components and produces: # -# ..-0.