diff --git a/.github/styles/config/vocabularies/PSDK/accept.txt b/.github/styles/config/vocabularies/PSDK/accept.txt index 9611e6dcb..ec68dd68d 100644 --- a/.github/styles/config/vocabularies/PSDK/accept.txt +++ b/.github/styles/config/vocabularies/PSDK/accept.txt @@ -33,9 +33,13 @@ Zink [Mm]ulticast [Rr]enderers? [Ss]carthgap +[Ss]ubgraphs? [Tt]oolchain balenaEtcher bdebstrap fdisk +logits? mmdebstrap +perfsim umount +venv diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f5d7aaff6..f689a5783 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,7 +5,9 @@ name: "build" on: # yamllint disable-line rule:truthy push: - branches: [master] + branches: [master, am62d-edgeai] + pull_request: + branches: [master, am62d-edgeai] concurrency: group: ${{ github.ref_name }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 07a533d1a..83f1c738c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -16,6 +16,8 @@ concurrency: permissions: contents: read + pages: write + id-token: write jobs: agregate: @@ -62,7 +64,11 @@ jobs: permissions: pages: write # Required for deployment to GitHub Pages id-token: write # Required for deployment to GitHub Pages + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} steps: - name: Update github page deployment + id: deployment uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 diff --git a/.rstcheck.cfg b/.rstcheck.cfg index 3c3e4dfd8..47e5e029e 100644 --- a/.rstcheck.cfg +++ b/.rstcheck.cfg @@ -3,5 +3,5 @@ ignore_directives = ifconfig,graphviz,grid,grid-item,grid-item-card,card,dropdow # ignore the following languages because they require external compilers and # snippets in this repo depend on external code ignore_languages = c,cpp,c++ -ignore_substitutions = __PART_FAMILY_NAME__,__PART_FAMILY_DEVICE_NAMES__,__PRODUCT_LINE_NAME__,__SDK_BUILD_MACHINE__,__SDK_FULL_NAME__,__SDK_SHORT_NAME__,__SDK_INSTALL_FILE__,__SDK_INSTALL_DIR__,__SDK_DOWNLOAD_URL__,__LINUX_UBUNTU_VERSION_LONG__,__LINUX_UBUNTU_VERSION_SHORT__,__OPTEE_PLATFORM_FLAVOR__,__RTOS_UBUNTU_VERSION_LONG__,__WINDOWS_SUPPORTED_LONG__,__FEATURINGMATRIX__,__SYSFW_CORE_NAME__,__PCIE_BASE_ADDRESS__,__PCIE_DEVICE_ID__,__IMAGE_TYPE__,__SDK_OS_NAME__ +ignore_substitutions = __PART_FAMILY_NAME__,__PART_FAMILY_DEVICE_NAMES__,__PRODUCT_LINE_NAME__,__SDK_BUILD_MACHINE__,__SDK_FULL_NAME__,__SDK_SHORT_NAME__,__SDK_INSTALL_FILE__,__SDK_INSTALL_DIR__,__SDK_DOWNLOAD_URL__,__LINUX_UBUNTU_VERSION_LONG__,__LINUX_UBUNTU_VERSION_SHORT__,__OPTEE_PLATFORM_FLAVOR__,__RTOS_UBUNTU_VERSION_LONG__,__WINDOWS_SUPPORTED_LONG__,__FEATURINGMATRIX__,__SYSFW_CORE_NAME__,__PCIE_BASE_ADDRESS__,__PCIE_DEVICE_ID__,__IMAGE_TYPE__,__SDK_OS_NAME__,__EDGEAI_SDK_VERSION__,trade ignore_messages = (Hyperlink target.*is not referenced\.) diff --git a/conf.py b/conf.py index c84b2b8c9..dc5b183b6 100644 --- a/conf.py +++ b/conf.py @@ -261,6 +261,12 @@ FAMILY = os.environ.get("DEVFAMILY", "") OS = os.environ.get("OS", "") + +# EdgeAI docs prefix every shell command with a "$ " prompt symbol; strip it +# from the copy-button output so copied text is directly runnable. +if OS == "edgeai" and FAMILY == "AM62DX": + copybutton_prompt_text = "$ " + try: globals().update(importlib.import_module(f"configs.{FAMILY}.{FAMILY}_{OS}_tags").__dict__) except ModuleNotFoundError as exc: diff --git a/configs/AM62DX/AM62DX_edgeai_config.txt b/configs/AM62DX/AM62DX_edgeai_config.txt new file mode 100644 index 000000000..66c278a67 --- /dev/null +++ b/configs/AM62DX/AM62DX_edgeai_config.txt @@ -0,0 +1,30 @@ +# General family replacement variables and configuration values for AM62D EdgeAI build + +# This file is processed by Python scripts to define both replacement +# variable and configuration values for a device family build. +# For replacement variables, all variables should be listed on a single +# line, and all the variables should be listed on consecutive lines +# (i.e. no additional blank lines between the replacement variables). +# Similarly, all the configuration values should be listed on individual, +# consecutive lines (with no blank lines between the configuration values). +# +# Note that neither replacement variables nor configuration values may contain +# a colon (i.e. ":") in their name. However, values may contain colons. + +Replacement Variables +--------------------- +'__PART_FAMILY_NAME__' : 'AM62Dx' +'__PART_FAMILY_DEVICE_NAMES__' : 'AM62Dx' +'__PRODUCT_LINE_NAME__' : 'Sitara MPU' +'__SDK_FULL_NAME__' : 'Processor SDK Linux EdgeAI AM62Dx' +'__SDK_SHORT_NAME__' : 'PSDK EdgeAI' +'__SDK_DOWNLOAD_URL__' : '`AM62D-SDK-Download-page `__' +'__EDGEAI_SDK_VERSION__' : '12.1' +'__TIDL_VERSION__' : '11_02_18_00' +'__TIDL_TOOLS_VERSION__' : '11.2.x' + +Configuration Values +-------------------- +'CONFIG_part_family' : 'AM62DX_family' +'CONFIG_part_variant' : 'AM62DX' +'CONFIG_sdk' : 'SITARA' diff --git a/configs/AM62DX/AM62DX_edgeai_tags.py b/configs/AM62DX/AM62DX_edgeai_tags.py new file mode 100644 index 000000000..9ea0dc882 --- /dev/null +++ b/configs/AM62DX/AM62DX_edgeai_tags.py @@ -0,0 +1,6 @@ +# Project name and HTML title +project = u'Processor SDK Linux Edge AI for AM62Dx' +html_title = 'Processor SDK Linux Edge AI for AM62Dx Documentation' + +# The master toctree document. +master_doc = 'devices/AM62DX/edgeai/index' diff --git a/configs/AM62DX/AM62DX_edgeai_toc.txt b/configs/AM62DX/AM62DX_edgeai_toc.txt new file mode 100644 index 000000000..8fe5720bc --- /dev/null +++ b/configs/AM62DX/AM62DX_edgeai_toc.txt @@ -0,0 +1,13 @@ +devices/AM62DX/edgeai/index +devices/AM62DX/edgeai/overview +devices/AM62DX/edgeai/evm_hardware_setup +devices/AM62DX/edgeai/out_of_box_demo +devices/AM62DX/edgeai/models_artifacts_and_setup +devices/AM62DX/edgeai/linux_audio_inference_app +devices/AM62DX/edgeai/getting_started_gstreamer +devices/AM62DX/edgeai/edgeai_model_workflow +devices/AM62DX/edgeai/model_zoo +devices/AM62DX/edgeai/model_compile_evaluate +devices/AM62DX/edgeai/model_inspect +devices/AM62DX/edgeai/integrating_new_model +devices/AM62DX/edgeai/performance diff --git a/configs/AM62DX/AM62DX_linux_toc.txt b/configs/AM62DX/AM62DX_linux_toc.txt index 49de07843..bd317a2d9 100644 --- a/configs/AM62DX/AM62DX_linux_toc.txt +++ b/configs/AM62DX/AM62DX_linux_toc.txt @@ -23,6 +23,7 @@ linux/Overview/GCC_ToolChain linux/Overview/Processor_SDK_Linux_create_SD_card linux/Foundational_Components +linux/index_Edge_AI linux/Foundational_Components_U-Boot linux/Foundational_Components/U-Boot/Users-Guide linux/Foundational_Components/U-Boot/Build-Guide-K3 @@ -125,5 +126,6 @@ linux/Demo_User_Guides/index_Demos linux/Demo_User_Guides/AM62D_Dsp_Offload_User_Guide linux/Demo_User_Guides/AM62D_2DFFT_User_Guide linux/Demo_User_Guides/AM62D_Cascade_Biquad_Parametric_EQ_User_Guide +linux/Demo_User_Guides/AM62D_EdgeAI_User_Guide linux/Documentation_Tarball diff --git a/source/devices/AM62DX/edgeai/edgeai_model_workflow.rst b/source/devices/AM62DX/edgeai/edgeai_model_workflow.rst new file mode 100644 index 000000000..94be55756 --- /dev/null +++ b/source/devices/AM62DX/edgeai/edgeai_model_workflow.rst @@ -0,0 +1,131 @@ +.. include:: + +.. _edgeai-introduction: + +######################## +EdgeAI Model Workflow +######################## + +This guide walks through the workflow for developing and deploying audio +deep-learning (DL) models on the |__PART_FAMILY_NAME__| platform, using +|__SDK_FULL_NAME__| |__EDGEAI_SDK_VERSION__|. It is written for embedded +developers who want to run, compile, evaluate, and benchmark audio models on the +AM62D - either out of the box on the target with pre-compiled artifacts, or +through the x86 compile/evaluate workflow. + +Four **reference models** serve as worked examples throughout the guide, but the +workflow is not limited to them: the same compile/evaluate/inspect flow is a +bring-your-own-model (BYOM) flow that accepts any supported `ONNX +`__ model. More reference models will be added over time. + +The remaining sections cover: + +- :ref:`EdgeAI Model Zoo ` - out-of-box demos on the AM62D + target (inference scripts and Jupyter notebooks) using pre-trained models and + pre-compiled artifacts. +- :ref:`Model Compile and Evaluate ` - the + x86 BYOM workflow built on ``edgeai-tidlrunner``: compile a model for the AM62D + and evaluate accuracy on a dataset. +- :ref:`Model Inspect ` - analyze the compiled subgraphs + and per-layer behavior with Model Inspector. +- :ref:`Performance benchmark ` - latency and memory + methodology with per-model tables. + +******************** +Development workflow +******************** + +The end-to-end path is the same for a reference model or your own model: + +#. **Obtain an ONNX model** - a reference graph shipped with the EdgeAI ModelZoo, + or your own exported ONNX. +#. **Compile** it for the C7\ |trade| NPU on an x86 host (:ref:`Model Compile and + Evaluate `). +#. **Evaluate** accuracy or audio-quality metrics on a dataset. +#. **Inspect** the compiled subgraphs and per-layer analysis (:ref:`Model + Inspect `). +#. **Run** on the AM62D target using the compiled artifact + (:ref:`EdgeAI Model Zoo `). +#. **Benchmark** latency and memory (:ref:`Performance `). + +**************** +Reference models +**************** + +Four reference models are provided across two task families, as worked examples +of the workflow. Each is delivered as an `ONNX `__ graph and, +except for GTCRN, is offloaded to the C7\ |trade| NPU through the TVM runtime +(TVM-RT); GTCRN is the one that runs ARM-only. See :ref:`the framework overview +` below for what that means. The same flow accepts your own +ONNX model, and more reference models will be added over time. + +Audio analysis and classification +================================= + +**VGGish11** (model ID ``ac-0101``) + Audio-clip classification. Input is a log-mel patch of shape ``(1, 1, 64, 126)``; + output is ``(1, 10)`` class logits plus a ``(1, 512)`` embedding. Runs at 8-bit, + fully offloaded to the C7\ |trade| NPU via TVM-RT (a single TIDL subgraph). + +**YAMNet** (model ID ``ac-0201``) + Audio-event classification. Input is a log-mel patch of shape ``(1, 1, 96, 64)``; + output is ``(1, 521)`` class scores. Runs at 8-bit, fully offloaded to the + C7\ |trade| NPU via TVM-RT (a single TIDL subgraph). + +Both classification models are evaluated on the UrbanSound8K dataset. + +Speech enhancement +================== + +**GCRN** (model ID ``ase-0201``) + Convolutional-recurrent speech enhancement (noise suppression) over a fixed 4-second + window. Input and output are complex spectrograms of shape ``(1, 2, 401, 161)``. + Runs at 16-bit and is offloaded to the C7\ |trade| NPU via TVM-RT + (``tidl_offload: true``). + +**GTCRN** (model ID ``ase-0100``) + Grouped temporal convolutional-recurrent speech enhancement. Input and output + are complex spectrograms of shape ``(1, 257, T, 2)`` with a dynamic time + dimension ``T``. Runs FP32 on the Arm core through the ONNX runtime + (``tidl_offload: false``) - it is the only supported model that is **not** + offloaded to the C7\ |trade| NPU. + +Both speech-enhancement models are evaluated on the VoiceBank-DEMAND-16k +dataset. + +.. _edgeai-tvm-tidl: + +******************** +TVM + TIDL framework +******************** + +The AM62D audio models are compiled and deployed with **TI TVM**, TI's fork of +the `Apache TVM `__ deep-learning compiler. The target +accelerator is the **C7**\ |trade| **NPU** - TI's Neural Processing Unit, which +combines the **C7x DSP** with the **Matrix Multiplication Accelerator (MMA)** +(referred to as *C7x+MMA* in older documentation). + +TI TVM integrates **TIDL** (TI Deep Learning) as an accelerator backend: +TIDL-supported layers are offloaded to the MMA, while TVM generates **C7x DSP** +code for any remaining layers. Because the MMA and the C7x DSP together make up +the C7\ |trade| NPU, the result is a single deployable artifact in which the +**entire model runs on the C7**\ |trade| **NPU**. + +Compilation runs on an x86 host and proceeds in stages: + +#. **Model import** - the ONNX graph is imported into TVM. +#. **Partitioning** - using the TIDL import API, the graph is split into + TIDL-supported subgraphs (compiled by the TIDL network compiler) and the + remaining operators (handled by TVM C7x DSP code generation). +#. **Artifact generation** - both parts are packaged into one TVM-RT deployable + artifact (``deploy_lib.so`` + ``deploy_graph.json`` + ``deploy_param.params``). + +At run time on the AM62D, the TVM runtime loads the artifact and dispatches the +offloaded subgraphs to TIDL-RT on the C7\ |trade| NPU. + +.. figure:: ../../../images/audioai/tvm_tidl_flow.svg + :width: 80 % + :align: center + + TVM + TIDL compile-and-deploy flow for the AM62D audio models. Green blocks + are handled by TIDL; white blocks are handled by TVM. diff --git a/source/devices/AM62DX/edgeai/evm_hardware_setup.rst b/source/devices/AM62DX/edgeai/evm_hardware_setup.rst new file mode 100644 index 000000000..5079b4f39 --- /dev/null +++ b/source/devices/AM62DX/edgeai/evm_hardware_setup.rst @@ -0,0 +1,229 @@ +.. _am62dx_edgeai_getting_started_platform: + +====================================== +Hardware Setup and Configuration +====================================== + +This section covers the hardware setup and software configuration required to prepare +the AM62D EVM for running EdgeAI inference applications. + +Hardware setup +============== + +.. _pub_edgeai_am62dx_evm_setup: + +AM62D EVM +--------- + +The AM62D EVM is a development board for audio AI applications built around the AM62DX +processor with a dedicated C7x™ DSP for neural network inference and digital signal +processing. + +To run the demos on the AM62D EVM you will require: + +- `AM62D EVM `__ +- Minimum 16GB SD card +- USB2.0 Type-C ports (5V‐15V input range) +- USB-to-UART cable for serial console access +- Audio input source (e.g., microphone) connected to J1/J2 3.5mm TRS Port +- Audio output device (e.g., headphones) connected to J3/J4 3.5mm TRS Port +- Required: Ethernet cable for remote access via `DP83867-EVM-AM `__ + +Connect the components to the AM62D EVM as shown below: + +.. figure:: /images/am62dx_board_config.png + :height: 600 + :width: 1000 + :align: center + + AM62D EVM layout showing connectors, switches, and boot mode selection + +Refer to the `AM62D Quick Start Guide `__ +for detailed information about AM62D EVM layout and component locations. For more information, visit AM62Dx EVM `User guide `_ + +Boot Mode Configuration +----------------------- + +Set the BOOTMODE switches to SD card boot mode before powering on: + +- BOOTMODE [8:15] (SW3) = 0100 0000 +- BOOTMODE [0:7] (SW2) = 1100 0010 + +.. figure:: /images/am62dx_boot_pin.png + :height: 500 + :width: 500 + :align: center + + AM62D EVM boot pin configuration + +Audio Connectors +---------------- + +- Connect audio input source to J1 and J2 TRS 3.5mm Jacks (PCM6240 ADC input) (Refer Fig: 2.1) +- Connect audio output to speakers/headphones via J3 and J4 TRS 3.5mm Jacks (TAD5212 DAC output) + +Software setup +============== + +Preparing SD Card Image +----------------------- + +Download the latest Processor SDK Linux AM62DX image from: + +|__SDK_DOWNLOAD_URL__| + +Look for the file: ``tisdk-edgeai-image-am62dxx-evm.rootfs.wic.xz`` + +Download Balena Etcher from https://www.balena.io/etcher/ and flash the image: + +.. figure:: /images/edgeai/balena_etcher.png + :scale: 100 + :align: center + + Balena Etcher SD card flashing tool + +1. Insert SD card into card reader connected to host PC +2. Launch Balena Etcher +3. Click "Select image" and choose the downloaded ``.wic.xz`` file +4. Click "Select target" and choose the SD card device +5. Click "Flash" to start flashing (this may take 10-15 minutes) +6. Wait for completion confirmation +7. Safely eject the SD card + +.. note:: + + Tested with Balena Etcher version 1.7.0. Some issues reported with version 1.7.2. + If flashing fails, try a different version or use ``dd`` command on Linux. + +If using a larger than 16GB SD card, expand the root filesystem to use the full capacity: + +.. code-block:: bash + + #find the SD card device entry using lsblk (Eg: /dev/sdc) + #Unmount the BOOT and rootfs partition before using parted tool + umount /dev/sdX1 + umount /dev/sdX2 + + #Use parted tool to resize the rootfs partition + parted -s /dev/sdX resizepart 2 '100%' + e2fsck -f /dev/sdX2 + resize2fs /dev/sdX2 + + #replace /dev/sdX in above commands with SD card device entry + +.. _pub_edgeai_am62dxx_flashing: + +Power ON and Boot +----------------- + +Ensure that the power supply is disconnected before inserting the SD card. +Once the SD card is firmly inserted and the AM62D EVM is powered ON, the board will +boot in 15-30 seconds. + +Monitor the UART console for boot messages: + +.. code-block:: console + + U-Boot SPL 2026.01-ti-g792d3f648d38 (Sep 29 2026 - 22:45:31 +0000) + SYSFW ABI: 4.0 (firmware rev 0x000c '12.1.2--v12.01.02 (Clever Cat)') + Set clock rates for '/a53@0', CPU: 1250MHz at Speed Grade 'T' + SPL initial stack usage: 13568 bytes + Trying to boot from MMC2 + Authentication passed + Authentication passed + Authentication passed + Authentication passed + Authentication passed + Starting ATF on ARM64 core... + + NOTICE: BL31: v2.15.0(release):v2.15.0 + NOTICE: BL31: Built : 16:29:49, May 28 2026 + + U-Boot SPL 2026.01-ti-g792d3f648d38 (Sep 29 2026 - 22:45:31 +0000) + SYSFW ABI: 4.0 (firmware rev 0x000c '12.1.2--v12.01.02 (Clever Cat)') + . + . + . + . + . + . + Type 'root' to login with superuser privileges (no password will be asked). + am62dxx-evm login: root + +At the login prompt, log in as the root user with no password: + +UART Serial Console +------------------- + +The AM62D EVM provides multiple serial ports via USB. 4 serial ports will come up on +your PC — connect to port 1 or equivalent to get the boot logs. + +For **Linux** use minicom: + +.. code-block:: bash + + sudo minicom -D /dev/ttyUSB1 -c on + +.. note:: + + If you are unable to input from keyboard, disable Hardware Flow Control in minicom: + Press Ctrl+A, then O (Setup), then S (Serial port), and verify Flow Control is off. + +For **Windows** use Tera Term: + +https://learn.sparkfun.com/tutorials/terminal-basics/tera-term-windows + +.. note:: + + Baud rate should be configured to 115200 bps. + +Connect Remotely +---------------- + +You can also access the AM62D EVM via SSH using the IP address shown on the console: + +.. code-block:: bash + + root@am62dxx-evm:~# ifconfig + +From your host PC: + +.. code-block:: console + + host# ssh root@ + +If using Ubuntu 22.04, add the following to ``~/.ssh/config``: + +.. code-block:: bash + + Host 192.168.* + HostKeyAlgorithms=+ssh-rsa + +For a full IDE experience, install Visual Studio Code with the "Remote Development" +extension pack from https://code.visualstudio.com/download and connect via Remote-SSH. + +Verifying the System +-------------------- + +After logging in, verify the system is properly booted: + +Check DSP state (should report 'running'): + +.. code-block:: console + + root@am62dxx-evm:~# cat /sys/class/remoteproc/remoteproc0/state + running + +List RPMsg devices: + +.. code-block:: console + + root@am62dxx-evm:~# ls /dev/rpmsg* + /dev/rpmsg0 /dev/rpmsg1 /dev/rpmsg2 /dev/rpmsg3 /dev/rpmsg_ctrl0 /dev/rpmsg_ctrl1 /dev/rpmsg_ctrl2 + +List DMA heap devices: + +.. code-block:: console + + root@am62dxx-evm:~# ls /dev/dma_heap + carveout_dsp_shared-memories carveout_edgeai_shared-memories default_cma_region linux,cma system diff --git a/source/devices/AM62DX/edgeai/getting_started_gstreamer.rst b/source/devices/AM62DX/edgeai/getting_started_gstreamer.rst new file mode 100644 index 000000000..b1a60d8a7 --- /dev/null +++ b/source/devices/AM62DX/edgeai/getting_started_gstreamer.rst @@ -0,0 +1,399 @@ +.. _pub_edgeai_am62dx_getting_started: + +######################### +GStreamer Audio Pipelines +######################### + +Introduction +============ + +The GStreamer pipeline framework on AM62DX enables real-time audio AI inference through two custom +plugins: ``tidspkernel`` and ``titvm``. Together, they offload signal processing and neural network +inference to the C7x™ DSP while orchestrating the data flow from Linux. With GStreamer, you can: + +- **Build Custom Pipelines** — Compose audio processing chains using standard GStreamer elements +- **DSP Offload** — Offload DSP signal chains (ex: STFT, ISTFT, and deinterleave/interleave) to the C7x™ DSP +- **TVM Inference** — Run TVM-compiled AI models on the C7x™ NPU via the TVM runtime +- **Flexible I/O** — Read from WAV files, live ALSA input, or other GStreamer sources +- **Direct Command-Line Control** — Use ``gst-launch-1.0`` to quickly test and prototype pipelines + +Supported Pipelines +=================== + +.. list-table:: + :header-rows: 1 + :widths: 30 30 20 20 + + * - Pipeline + - Use Case + - Default Input + - Output + * - Speech enhancement (GCRN) + - Noise reduction with spectral reconstruction + - ``input_audio.wav`` + - Enhanced WAV file (playback or file) + * - Audio classification (YAMNet) + - 521-class audio event detection + - ``urbansound_16k.wav`` + - Top-k class names and scores (console) + * - Audio classification (VGGish) + - 10-class environmental sound classification + - ``urbansound_16k.wav`` + - Top-k class names and scores (console) + +Prerequisites +============= + +All runtime libraries, model artifacts, sample audio files, model daemon, and DSP firmware +verification are pre-packaged in the AM62DX EdgeAI image. See +:ref:`Models, Artifacts and Setup ` for details. + +To build the GStreamer plugins from source, the following host tools are required: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Tool + - Purpose + * - ``meson`` / ``ninja`` + - Build system for GStreamer plugins + * - ARM cross-compiler + - Required only for host-side cross-compilation + * - ``pkg-config`` + - Dependency resolution during build + + +Native Compilation & Installation on AM62D EVM +---------------------------------------------- + +The plugins are pre-installed on the AM62DX EdgeAI image. If you need to rebuild them +natively on the AM62D EVM, follow the steps below: + +.. code-block:: console + + root@am62dxx-evm:~# cd /opt/edgeai-gst-plugins/ + root@am62dxx-evm:/opt/edgeai-gst-plugins# export SOC=am62d + root@am62dxx-evm:/opt/edgeai-gst-plugins# meson setup build --prefix=/usr -Dpkg_config_path=pkgconfig \ + -Dtvm-plugins=enabled -Dstft-plugins=enabled \ + -Ddl-plugins=disabled -Denable-tidl=disabled + root@am62dxx-evm:/opt/edgeai-gst-plugins# ninja -C build + root@am62dxx-evm:/opt/edgeai-gst-plugins# ninja -C build install + root@am62dxx-evm:/opt/edgeai-gst-plugins# ldconfig + + +Verifying the Installation +-------------------------- + +Execute the following command to verify the tidspkernel GStreamer plugin is installed: + +.. code-block:: console + + root@am62dxx-evm:~# gst-inspect-1.0 tidspkernel + Factory Details: + Rank none (0) + Long-name TI DSP Kernel Transform + Klass Transform/DSP + Description Generic DSP kernel offload (STFT/ISTFT/Deinterleave/Interleave) via RPMsg-DMA + Author Pratham Deshmukh + + Plugin Details: + Name ti + Description GStreamer plugin for TI devices + Filename /usr/lib/gstreamer-1.0/libgstti.so + Version 0.7.0 + License Proprietary + Source module GstTIOVX + Binary package GstTIOVX source release + Origin URL http://ti.com + + GObject + +----GInitiallyUnowned + +----GstObject + +----GstElement + +----GstBaseTransform + +----GstDspKernel + +Execute the following command to verify the titvm GStreamer plugin is installed: + +.. code-block:: console + + root@am62dxx-evm:~# gst-inspect-1.0 titvm + Factory Details: + Rank none (0) + Long-name TI TVM Inference + Klass Transform/ML + Description TVM inference on TI processors with C7x™ DSP acceleration + Author Pratham Deshmukh + + Plugin Details: + Name ti + Description GStreamer plugin for TI devices + Filename /usr/lib/gstreamer-1.0/libgstti.so + Version 0.7.0 + License Proprietary + Source module GstTIOVX + Binary package GstTIOVX source release + Origin URL http://ti.com + + GObject + +----GInitiallyUnowned + +----GstObject + +----GstElement + +----GstBaseTransform + +----GstTiTvm + + +Both commands must print the element details. If an element is missing, check that +:file:`/usr/lib/gstreamer-1.0/libgstti.so` exists and that :command:`ldconfig` has run. + +.. _pub_edgeai_am62dx_model_artifacts: + +GCRN based Speech Enhancement +============================= + +Speech enhancement removes background noise from input audio source while keeping the speech +intelligible — useful for voice calls, meeting recordings, hearables, and any product that +captures speech in a noisy environment. This pipeline runs that speech enhancement full +pipeline on the board: it reads a 16 kHz mono WAV file, converts it to the frequency domain on +the DSP, runs the GCRN model on the result, and converts the enhanced spectrum back to audio +which can be played or saved. + +.. _pub_edgeai_am62dx_enhancement_command: + +Speech Enhancement Pipeline using GCRN +-------------------------------------- + +Execute the following command to run the speech enhancement pipeline. The command writes ``/tmp/gst_enhanced.wav`` and plays the enhanced speech output through ``autoaudiosink``. Connect headphones to J3/J4 as directed in :ref:`pub_edgeai_am62dx_evm_setup` to listen to the enhanced speech. + +.. code-block:: bash + + INPUT=/usr/share/tvm_inference/input/input_audio.wav + MODEL=/usr/share/tvm_inference/artifacts/gcrn + + gst-launch-1.0 -v \ + filesrc location=$INPUT ! wavparse ! \ + audio/x-raw,format=S16LE,rate=16000,channels=1 ! \ + tidspkernel name=stft msg-type=0x1020 model-path=$MODEL \ + hop-size=160 fft-size=320 window-frames=401 batch-size=64 \ + max-stream-samples=160480 ! \ + tidspkernel name=deinterleave msg-type=0x1040 interleave_direction=0 \ + model-path=$MODEL fft-size=320 window-frames=401 ! \ + titvm model-path=$MODEL ! \ + tidspkernel name=interleave msg-type=0x1040 interleave_direction=1 \ + model-path=$MODEL fft-size=320 window-frames=401 ! \ + tidspkernel name=istft msg-type=0x1030 model-path=$MODEL \ + hop-size=160 fft-size=320 window-frames=401 batch-size=64 \ + max-stream-samples=160480 ! \ + audio/x-raw,format=S16LE,rate=16000,channels=1 ! \ + tee name=t \ + t. ! queue ! wavenc ! filesink location=/tmp/gst_enhanced.wav \ + t. ! queue ! audioconvert ! audioresample ! autoaudiosink + +For a headless run, remove the ``tee`` branch and keep only the file output: + +.. code-block:: bash + + ... ! audio/x-raw,format=S16LE,rate=16000,channels=1 ! \ + wavenc ! filesink location=/tmp/gst_enhanced.wav + +.. figure:: /images/am62dx/edgeai/pipeline_gcrn.png + :scale: 75 + :align: center + + GStreamer data-flow for GCRN speech enhancement: input WAV → STFT → GCRN model inference → ISTFT → output WAV + playback + +.. _pub_edgeai_am62dx_enhancement_chunking: + +.. _pub_edgeai_am62dx_audio_classification: + +YAMNet & VGGish based Audio Classification +========================================== + +Audio classification identifies *what* sound is present in input audio source — a siren, glass +breaking, a dog barking, a machine running — rather than transcribing speech. That makes it +useful for acoustic event detection, environmental and industrial sound monitoring, and +safety or anomaly alerts triggered by sound rather than video. This pipeline runs YAMNet or +VGGish on a 16 kHz mono WAV file on the DSP, and prints the top-k predicted classes and their +scores for each analysis window to the console as the pipeline runs. + +YAMNet and VGGish differ mainly in scope: YAMNet recognizes 521 general sound classes, while +VGGish is a smaller, 10-class model. Pick YAMNet for broad coverage, or VGGish when you only +need to distinguish a small, fixed set of sounds and want a lighter model. + +Audio Classification Pipeline using YAMNet +------------------------------------------ + +Execute the following command to run GStreamer based YAMNet audio classification pipeline: + +.. code-block:: bash + + gst-launch-1.0 -v filesrc location=/usr/share/tvm_inference/input/urbansound_16k.wav ! \ + wavparse ! audio/x-raw,format=S16LE,rate=16000,channels=1 ! \ + tidspkernel name=stft msg-type=0x1020 \ + model-path=/usr/share/tvm_inference/artifacts/yamnet \ + hop-size=160 window-frames=96 batch-size=64 ! \ + titvm model-path=/usr/share/tvm_inference/artifacts/yamnet \ + class-map-path=/usr/share/tvm_inference/labels/yamnet_label_list.txt \ + top-k=3 ! \ + fakesink + +YAMNet Command Output +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: text + + root@am62dxx-evm:~# gst-launch-1.0 -v filesrc location=/usr/share/tvm_inference/input/urbansound_16k.wav ! \ + Setting pipeline to PAUSED ... + [TVM] Live top-3 prediction printing enabled (521 classes from /usr/share/tvm_inference/labels/yamnet_label_list.txt) + eptdev = rpmsg-char-8-1455 + Created endpt device rpmsg-char-8-1455, fd = 7 port = 1027 + Pipeline is PREROLLING ... + /GstPipeline:pipeline0/GstWavParse:wavparse0.GstPad:src: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + /GstPipeline:pipeline0/GstCapsFilter:capsfilter0.GstPad:src: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + /GstPipeline:pipeline0/GstDspKernel:stft.GstPad:src: caps = application/octet-stream + /GstPipeline:pipeline0/GstTiTvm:titvm0.GstPad:src: caps = application/octet-stream + /GstPipeline:pipeline0/GstFakeSink:fakesink0.GstPad:sink: caps = application/octet-stream + /GstPipeline:pipeline0/GstTiTvm:titvm0.GstPad:sink: caps = application/octet-stream + /GstPipeline:pipeline0/GstDspKernel:stft.GstPad:sink: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + /GstPipeline:pipeline0/GstCapsFilter:capsfilter0.GstPad:sink: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + Pipeline is PREROLLED ... + Setting pipeline to PLAYING ... + Redistribute latency... + New clock: GstSystemClock + Top-3 predictions for window 1: + 1. Speech: 0.532749 + 2. Music: -4.261989 + 3. Narration, monologue: -4.794737 + Top-3 predictions for window 2: + 1. Music: -1.598246 + 2. Speech: -2.663743 + 3. Television: -3.729240 + Top-3 predictions for window 3: + 1. Music: -1.065497 + 2. Animal: -2.663743 + 3. Speech: -3.196491 + Top-3 predictions for window 4: + 1. Speech: -1.598246 + 2. Music: -1.598246 + 3. Country: -5.860234 + Top-3 predictions for window 5: + 1. Speech: -2.130994 + 2. Music: -3.196491 + 3. Silence: -4.261989 + Top-3 predictions for window 6: + 1. Silence: 14.916960 + 2. Speech: -11.187720 + 3. Inside, small room: -15.982457 + Top-3 predictions for window 7: + 1. Silence: 12.253218 + 2. Speech: -10.122223 + 3. Inside, small room: -15.449709 + Top-3 predictions for window 8: + 1. Silence: 15.449709 + 2. Speech: -10.654972 + 3. Inside, small room: -14.916960 + Got EOS from element "pipeline0". + Execution ended after 0:00:00.302017695 + Setting pipeline to NULL ... + Freeing pipeline ...`` + +.. figure:: /images/am62dx/edgeai/pipeline_yamnet.png + :scale: 75 + :align: center + + GStreamer data-flow for YAMNet based audio classification: input WAV → STFT → YAMNet model inference → console output + +Audio Classification Pipeline using VGGish +------------------------------------------ + +Execute the following command to run GStreamer based VGGish audio classification pipeline: + +.. code-block:: bash + + gst-launch-1.0 -v filesrc location=/usr/share/tvm_inference/input/urbansound_16k.wav ! \ + wavparse ! audio/x-raw,format=S16LE,rate=16000,channels=1 ! \ + tidspkernel name=stft msg-type=0x1020 \ + model-path=/usr/share/tvm_inference/artifacts/vggish \ + hop-size=512 window-frames=126 batch-size=64 ! \ + titvm model-path=/usr/share/tvm_inference/artifacts/vggish \ + class-map-path=/usr/share/tvm_inference/labels/vggish_label_list.txt \ + top-k=3 ! \ + fakesink + +VGGish Command Output +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: text + + Setting pipeline to PAUSED ... + [TVM] Live top-3 prediction printing enabled (10 classes from /usr/share/tvm_inference/labels/vggish_label_list.txt) + eptdev = rpmsg-char-8-1295 + Created endpt device rpmsg-char-8-1295, fd = 7 port = 1027 + Pipeline is PREROLLING ... + /GstPipeline:pipeline0/GstWavParse:wavparse0.GstPad:src: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + /GstPipeline:pipeline0/GstCapsFilter:capsfilter0.GstPad:src: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + /GstPipeline:pipeline0/GstDspKernel:stft.GstPad:src: caps = application/octet-stream + /GstPipeline:pipeline0/GstTiTvm:titvm0.GstPad:src: caps = application/octet-stream + /GstPipeline:pipeline0/GstFakeSink:fakesink0.GstPad:sink: caps = application/octet-stream + /GstPipeline:pipeline0/GstTiTvm:titvm0.GstPad:sink: caps = application/octet-stream + /GstPipeline:pipeline0/GstDspKernel:stft.GstPad:sink: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + /GstPipeline:pipeline0/GstCapsFilter:capsfilter0.GstPad:sink: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + Pipeline is PREROLLED ... + Setting pipeline to PLAYING ... + Redistribute latency... + New clock: GstSystemClock + Top-3 predictions for window 1: + 1. Street music: 8.411190 + 2. Children playing: 8.206039 + 3. Air conditioner: -4.513321 + Got EOS from element "pipeline0". + Execution ended after 0:00:00.082239915 + Setting pipeline to NULL ... + Freeing pipeline ... + + +.. figure:: /images/am62dx/edgeai/pipeline_vggish.png + :scale: 75 + :align: center + + GStreamer data-flow for VGGish based audio classification: input WAV → STFT → VGGish model inference → console output + +Troubleshooting +=============== + +.. list-table:: + :header-rows: 1 + :widths: 40 60 + + * - Issue + - Resolution + * - ``max-stream-samples must be explicitly set (> 0)`` + - Set ``max-stream-samples`` on the STFT element. + * - ``padded length ... exceeds dma_input capacity`` + - Increase ``max-stream-samples`` to the padded length. + * - ``Invalid incoming buffer`` + - The input is longer than ``max-stream-samples``. Increase it. + * - ``Model daemon handshake failed`` followed by a long wait + - Start the daemon, or accept the in-process load time. + * - ``c7x_client_open() failed — is the c7x_compute DSP firmware running?`` + - The DSP firmware is not running. Check ``/sys/class/remoteproc/remoteproc0/state``. + If it shows ``offline``, reboot the evm. After the reboot, + rerun ``/sys/class/remoteproc/remoteproc0/state`` it should now show ``running`` + * - ``DMA heap alloc failed`` + - Not enough contiguous memory for the model. Reboot and retry, then check ``/proc/meminfo``. + * - ``TI-Offload exchange failed`` or ``DSP kernel failed: remote status=`` + - The DSP firmware does not match the plugin. Check the DSP firmware version. + * - ``Failed to set pipeline to PAUSED`` + - Run again with ``GST_DEBUG=tidspkernel:6,titvm:6`` and check the first error. + * - ``Failed to load class-map-path`` + - Check the path, and that the file is readable. + * - Banner prints but no predictions appear + - Run with ``GST_DEBUG=titvm:5``. Check that the pipeline reached PLAYING. + * - Predictions do not match the expected labels + - Check that the model and label file are a pair: YAMNet with ``yamnet_label_list.txt``, + VGGish with ``vggish_label_list.txt``. + * - ``top-k`` rejected + - Use a value from 1 to 521. + * - Long wait before banner prints + - ``titvm`` is waiting for the model daemon. Start it or accept the in-process load time. diff --git a/source/devices/AM62DX/edgeai/index.rst b/source/devices/AM62DX/edgeai/index.rst new file mode 100644 index 000000000..e77c24329 --- /dev/null +++ b/source/devices/AM62DX/edgeai/index.rst @@ -0,0 +1,30 @@ +.. _edgeai-index: + +###################################################### +Processor SDK Linux Edge AI Software Developer’s Guide +###################################################### + +.. rubric:: Welcome to the AM62D EdgeAI Software Developer's Guide + +.. toctree:: + :maxdepth: 5 + :numbered: + + overview + evm_hardware_setup + out_of_box_demo + models_artifacts_and_setup + linux_audio_inference_app + getting_started_gstreamer + edgeai_model_workflow + model_zoo + model_compile_evaluate + model_inspect + integrating_new_model + performance + +| + ++--------------------------------+--------------------------------------------------------------------------------------------------+ +| .. Image:: /images/E2e.jpg | For technical support please post your questions at `http://e2e.ti.com `__. | ++--------------------------------+--------------------------------------------------------------------------------------------------+ diff --git a/source/devices/AM62DX/edgeai/integrating_new_model.rst b/source/devices/AM62DX/edgeai/integrating_new_model.rst new file mode 100644 index 000000000..d20d254a0 --- /dev/null +++ b/source/devices/AM62DX/edgeai/integrating_new_model.rst @@ -0,0 +1,694 @@ +.. _am62dx_edgeai_integrating_new_model: + +####################### +Integrating a New Model +####################### + +This page covers what is needed to integrate a new deep learning model into the AM62DX +Edge AI system. Compiling the model and updating the DSP firmware are required for all +use cases. After that, you can run the model using either the Linux inference application +or GStreamer pipelines — both are independent paths. + +******** +Overview +******** + +- **Compile the Model** — required; generate TVM artifacts for the C7x™ DSP +- **DSP Firmware Changes** — required; register the model in the DSP signal chain +- **Linux Application** — use the ``rpmsg_inference_example`` application +- **GStreamer** — use ``tidspkernel`` and ``titvm`` GStreamer elements + +***************** +Compile the Model +***************** + +Use the TVM compilation workflow to produce three artifacts for the C7x™ DSP: + +- ``deploy_graph.json`` — graph description with input/output shapes +- ``deploy_lib.so`` — compiled model library +- ``deploy_param.params`` — model weights + +See :ref:`Model Compile and Evaluate `. + +******************** +DSP Firmware Changes +******************** + +The DSP firmware selects models at runtime via a ``ModelId`` enum and a dispatcher +``switch``. Adding a new model means adding one entry to the enum and one ``case`` to +the dispatcher. The full step-by-step guide is in the MCU+ SDK Edge AI Add New Model +documentation. + +Files to modify or create +========================= + +.. list-table:: + :header-rows: 1 + :widths: 45 55 + + * - File + - Change + * - ``common/model_config.h`` + - Add ``MODEL_NEWMODEL`` to the ``ModelId`` enum before ``MODEL_COUNT``; bump + ``MODEL_COUNT`` + * - ``common/models//TISP__signal_chain_memory_map.hpp`` + - New file: STFT/log-mel parameter macros (FFT size, hop size, channel count, + buffer sizes) + * - ``common/models//stft_signal_chain_.hpp`` + - New file: declare ``StftSignalChain`` class interface + * - ``common/models//stft_signal_chain_.cpp`` + - New file: implement the signal chain, ``SignalChainOps`` table, and + ``extern "C" newmodel_get_ops()`` + * - ``common/signal_chain_dispatch.cpp`` + - Add ``extern "C"`` declaration and new ``case MODEL_NEWMODEL`` in + ``signal_chain_get_ops()`` + * - ``am62dx-evm/.../example.projectspec`` + - Add ``-I`` include path and ```` entry for the new model source + * - ``am62dx-evm/.../makefile`` + - Add to ``FILES_cpp_common``, ``FILES_PATH_cpp_common``, and ``INCLUDES_common`` + +Key functions to implement in ``stft_signal_chain_.cpp`` +=============================================================== + +- ``create_pre_graph(int16_t *input_addr)`` — allocate arena buffers, generate STFT + analysis window (or log-mel filterbank), build a ``TISP::opVec`` of + ``TypeConversion`` + ``Stft`` (or ``LogMelSpectrum``) +- ``execute_pre_graph(void *pre_graph, int16_t *input_addr)`` — bind input address via + ``setAddr(..., TISP::kAddrModeIn)`` and call ``TISP::execute::graph()`` +- ``delete_pre_graph(void *pre_graph)`` — delete the ``TISP::opVec`` +- ``create_post_graph`` / ``execute_post_graph`` / ``delete_post_graph`` — only for + reconstruction-style models; set to ``NULL`` in ``SignalChainOps`` for analysis-only + models +- ``newmodel_get_preprocess_buf()`` / ``newmodel_get_postprocess_buf()`` — return arena + buffer addresses +- ``newmodel_reset_arenas()`` — call ``signal_chain_arena_reset_ready()`` +- ``newmodel_get_ops()`` — expose the static ``SignalChainOps`` table + +Two model types are supported +============================= + +- **Reconstruction-style** (e.g. DCCRN, GTCRN, GCRN): STFT pre-graph + ISTFT + post-graph; implement all six methods +- **Analysis-only** (e.g. VGGish, YAMNet): STFT/log-mel pre-graph only; post-graph + methods are ``NULL`` in the ``SignalChainOps`` table + +******************* +Linux Application +******************* + +Using a New Model with an Existing Pipeline Type +================================================ + +To bring your own model, first compile it for the AM62DX C7x™ NPU using the +:ref:`Model Compile and Evaluate ` workflow, which produces +the ``deploy_graph.json``, ``deploy_lib.so`` and ``deploy_param.params`` artifacts. + +Once you have the compiled artifacts: + +#. Deploy the model artifacts to the target: + + .. code-block:: console + + root@am62dxx-evm:~# mkdir -p /usr/share/tvm_inference/artifacts/my_model/ + root@am62dxx-evm:~# cp deploy_graph.json deploy_lib.so deploy_param.params \ + /usr/share/tvm_inference/artifacts/my_model/ + +#. Copy the nearest existing JSON pipeline file as a starting point and adjust it for + your model. The JSON files in ``example/edge-ai/json_files/`` are the authoritative + reference. Key fields to update: + + .. code-block:: json + + { + "pipeline_type": "audio_classification", + "input_file": "/usr/share/tvm_inference/input/my_audio.wav", + "artifacts_path": "/usr/share/tvm_inference/artifacts/my_model", + "labels_path": "/usr/share/tvm_inference/labels/my_model_labels.txt", + "num_classes": 10, + "dsp_config": { + "proc_id": 8, + "endpoint": 13 + }, + "stages": [ + { + "stage_id": "stft_analysis", + "service": "dsp", + "message_type": "C7X_MSG_STFT_ANALYZE", + "parameters": { + "selected_model": 4, + "hop_size": 160, + "model_elems": 64, + "total_frames": 96, + "batch_n": 64 + } + }, + { + "stage_id": "tvm_inference", + "service": "tvm", + "message_type": "TVM_INFERENCE", + "parameters": { + "input_shape": "1,1,96,64" + } + } + ] + } + + Keep ``proc_id``, ``endpoint``, and DSP ``message_type`` values consistent with + the deployed DSP firmware. Adjust ``selected_model``, ``model_elems``, ``total_frames`` + and ``input_shape`` to match your model's spectral layout. + +#. If your model outputs classifications, provide a labels file (one label per line): + + .. code-block:: console + + root@am62dxx-evm:~# cat > /usr/share/tvm_inference/labels/my_model_labels.txt << EOF + Label 0 + Label 1 + ... + EOF + +#. Run the application with the new pipeline: + + .. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example /path/to/my_pipeline.json \ + --input-file /path/to/test_audio.wav + +Adding a New Pipeline Type in C++ +================================== + +If the existing pipeline types (``speech_enhancement``, ``audio_classification``, +``stft_istft``, ``tvm_only``) do not cover your use case, you can add a new pipeline +type. The dispatch is in ``pipeline_manager.cpp`` and follows the same pattern for +every pipeline. + +**Add a header and source file** + +Create ``example/edge-ai/include/my_pipeline.h``: + +.. code-block:: cpp + + #pragma once + #include "pipeline_manager.h" + #include "dsp_task_client.h" + #include "tvm_inference_client.h" + + PipelineManager::CommandResult run_my_pipeline( + PipelineManager::State& state, + DspTaskClient& dsp_client, + TvmInferenceClient& tvm_client, + bool debug); + +Create ``example/edge-ai/src/my_pipeline.cpp``: + +.. code-block:: cpp + + #include "my_pipeline.h" + #include "pipeline_common.h" + #include + + PipelineManager::CommandResult run_my_pipeline( + PipelineManager::State& state, + DspTaskClient& dsp_client, + TvmInferenceClient& tvm_client, + bool debug) + { + // Access pipeline config and stages from state + const auto& config = state.pipeline_config; + const auto& stages = config.stages; + + // Find your custom stage by message_type + const PipelineManager::PipelineStage* my_stage = nullptr; + for (const auto& s : stages) { + if (s.message_type == "MY_CUSTOM_MSG") { + my_stage = &s; + break; + } + } + if (!my_stage) + throw PipelineError{"my_pipeline requires a MY_CUSTOM_MSG stage"}; + + // Read stage parameters + size_t hop_size = std::stoul(my_stage->parameters.at("hop_size")); + + // ... run DSP and TVM stages using dsp_client and tvm_client ... + + std::cout << "[App] my_pipeline complete" << std::endl; + return PipelineManager::CommandResult::SUCCESS; + } + +**Register the pipeline type in** ``pipeline_manager.cpp`` + +In ``run_from_json_file()``, add your new ``pipeline_type`` string to the dispatch +block alongside the existing types: + +.. code-block:: cpp + + #include "my_pipeline.h" // add this include at the top + + // ... inside run_from_json_file(), in the dispatch block: + if (pipeline_type == "speech_enhancement") { + result = run_speech_enhancement_pipeline(state_, *generic_client_, *tvm_client_, debug_); + } else if (pipeline_type == "audio_classification") { + result = run_audio_classification_pipeline(state_, *generic_client_, *tvm_client_, debug_); + } else if (pipeline_type == "tvm_only") { + result = run_tvm_pipeline(state_, *tvm_client_); + } else if (pipeline_type == "stft_istft") { + result = run_stft_istft_pipeline(state_, *generic_client_, debug_); + } else if (pipeline_type == "my_pipeline") { // add this + result = run_my_pipeline(state_, *generic_client_, *tvm_client_, debug_); + } else { + std::cout << "[App] Error: Unknown pipeline_type: " << pipeline_type << std::endl; + return -1; + } + +**Add the source file to** ``CMakeLists.txt`` + +In ``example/edge-ai/CMakeLists.txt``, add ``src/my_pipeline.cpp`` to the +``EDGE_AI_SOURCES`` list: + +.. code-block:: cmake + + set(EDGE_AI_SOURCES + src/main.cpp + src/tvm_inference_client.cpp + src/dsp_task_client.cpp + src/pipeline_manager.cpp + src/pipeline_common.cpp + src/audio_utils.cpp + src/speech_enhancement_pipeline.cpp + src/audio_classification_pipeline.cpp + src/tvm_pipeline.cpp + src/stft_istft_pipeline.cpp + src/my_pipeline.cpp # add this + ) + +**Create a JSON pipeline file for the new type** + +.. code-block:: json + + { + "pipeline_type": "my_pipeline", + "input_file": "/usr/share/tvm_inference/input/my_audio.wav", + "artifacts_path": "/usr/share/tvm_inference/artifacts/my_model", + "dsp_config": { "proc_id": 8, "endpoint": 13 }, + "stages": [ + { + "stage_id": "my_stage", + "service": "dsp", + "message_type": "MY_CUSTOM_MSG", + "parameters": { "hop_size": "160" } + }, + { + "stage_id": "tvm_inference", + "service": "tvm", + "message_type": "TVM_INFERENCE", + "parameters": { "input_shape": "1,1,96,64" } + } + ] + } + +Rebuild with ``cmake --build build`` and run: + +.. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example /path/to/my_pipeline.json + +Debugging and Troubleshooting +============================== + +**System checks** + +.. code-block:: console + + root@am62dxx-evm:~# grep . /sys/class/remoteproc/remoteproc*/name + root@am62dxx-evm:~# grep . /sys/class/remoteproc/remoteproc*/state + root@am62dxx-evm:~# ls -l /dev/rpmsg* /dev/dma_heap/* + root@am62dxx-evm:~# ldd /usr/bin/rpmsg_inference_example + root@am62dxx-evm:~# find /usr/share/tvm_inference -maxdepth 3 -type f + +**DSP firmware trace** + +.. code-block:: console + + root@am62dxx-evm:~# cat /sys/kernel/debug/remoteproc/remoteproc*/trace0 + +**Enable debug logging** + +.. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example --debug + +.. list-table:: + :header-rows: 1 + :widths: 40 60 + + * - Symptom + - Action + * - ``rpmsg_inference_example`` not found + - Check binary is in ``/usr/bin/`` and has execute permission. + Run ``ldd`` to check for missing shared libraries. + * - Connection to DSP endpoint fails + - Verify DSP state is ``running``. Check ``/dev/rpmsg*`` exists. + View DSP trace with ``trace0``. + * - Model artifact path not found + - Verify ``artifacts_path`` in the JSON matches the deployed directory. + Check ``deploy_graph.json``, ``deploy_lib.so`` and ``deploy_param.params`` are present. + * - Audio input device not found + - Run ``arecord -l`` to list ALSA devices. Verify ``--device`` name matches. + * - Long wait before inference starts + - ``tvm_model_daemon`` is not running. Start it or accept the in-process load time. + +********* +GStreamer +********* + +.. _pub_edgeai_am62dx_byom: + +Bring-Your-Own Model +==================== + +The three reference pipelines use pre-trained, pre-compiled models so you can see the +plugins working immediately. Most production use cases, though, need a model trained on +your own data — your own set of sound events, or a speech-enhancement model tuned to +your product's microphone and acoustic environment. This section covers what it takes to +run such a model through ``tidspkernel`` and ``titvm`` once it has been compiled for the +AM62Dx C7x™ DSP. + +Compiling the model itself — taking an ONNX graph through quantization and generating +``deploy_graph.json``, ``deploy_lib.so`` and ``deploy_param.params`` — is covered +separately in :ref:`Model Compile and Evaluate `. This section +assumes you already have those three artifacts and focuses on wiring them into a GStreamer +pipeline. + +Requirements +------------ + +A compiled model directory with the same three files the reference models use: + +.. code-block:: text + + my_model/ + ├── deploy_graph.json graph description; input/output shapes read from attrs.shape + ├── deploy_lib.so compiled model library + └── deploy_param.params model weights + +Beyond these files, you must specify two critical model properties that the plugins cannot +auto-detect: + +- **Task type**: whether the model performs speech enhancement (like GCRN) or + classification (like YAMNet/VGGish). This determines which pipeline you should adapt. +- **Spectral layout**: the FFT size, hop size, and window length the model was trained + with, plus the number of spectral coefficients per frame that the DSP firmware must + produce. + +Example: A New Model +-------------------- + +Suppose you have a new model with a unique architecture and spectral front-end, and +DSP firmware ModelId ``5`` has already been added to support it. To use it in a GStreamer +pipeline, follow the same pattern as the YAMNet/VGGish examples, but explicitly set all +spectral properties instead of relying on path-based auto-detection: + +.. code-block:: bash + + MODEL=/path/to/my_model + gst-launch-1.0 -v filesrc location=$INPUT ! \ + wavparse ! audio/x-raw,format=S16LE,rate=16000,channels=1 ! \ + tidspkernel name=stft msg-type=0x1020 model-path=$MODEL \ + selected-model=5 model-elems=128 hop-size=160 fft-size=256 \ + window-frames=100 batch-size=64 ! \ + titvm model-path=$MODEL \ + class-map-path=/path/to/my_model/labels.txt top-k=3 ! \ + fakesink + +This command uses the same approach as the reference models — the difference is that all +spectral parameters (``selected-model``, ``model-elems``, ``fft-size``, ``hop-size``, +``window-frames``) are now explicitly set for your model rather than auto-detected from +the path. The ``titvm`` element requires no modifications beyond ``model-path`` — it +automatically loads and runs whatever TVM artifacts it finds in that directory. + +For a speech-enhancement model with a new architecture, use the same approach: build +your pipeline with the ``deinterleave`` → ``tidspkernel`` (STFT) → ``titvm`` → +``interleave`` → ``tidspkernel`` (ISTFT) chain from the GCRN reference pipeline, then +set the ``tidspkernel`` properties to match your model's spectral layout instead of +GCRN's. + +Supporting New Model Architectures in the Plugins +================================================== + +If your model introduces a new architecture or spectral processing scheme not covered by +existing DSP firmware ModelIds, the DSP DSP firmware must first add support for that new +ModelId. Once the DSP firmware is updated, the GStreamer plugins can immediately wire it +in by setting the ``selected-model`` property to the new DSP firmware ID. + +However, if you need to extend the plugins themselves to better support a new model +category or add automatic detection logic, the following sections show the code you +would modify in the GStreamer plugin source files. Section :ref:`pub_edgeai_am62dx_building_plugin` +shows how to build the new modified plugin. + +Adding a New GStreamer Property to ``tidspkernel`` +-------------------------------------------------- + +If your new model requires an additional configuration parameter (for example, a +``model-specific-param`` flag), add it to the plugin header and implementation: + +**1. Header: ``gstdspkernel.h`` - Add to the struct:** + +.. code-block:: c + + struct _GstDspKernel + { + /* ... existing properties ... */ + guint fft_size; + guint window_frames; + guint selected_model; + guint model_elems; + + /* NEW: Add your custom property */ + guint model_specific_param; /* New parameter for custom model */ + }; + +**2. Implementation: ``gstdspkernel.cpp`` - Add property enum:** + +.. code-block:: c + + /* Property IDs */ + enum + { + PROP_0, + PROP_MSG_TYPE, + PROP_INTERLEAVE_DIRECTION, + PROP_HOP_SIZE, + PROP_FFT_SIZE, + PROP_WINDOW_FRAMES, + PROP_BATCH_SIZE, + PROP_SELECTED_MODEL, + PROP_MODEL_ELEMS, + PROP_MODEL_PATH, + PROP_OVERLAP_FRAMES, + PROP_CHUNKING_MODE, + PROP_MAX_STREAM_SAMPLES, + + /* NEW: Add your property ID */ + PROP_MODEL_SPECIFIC_PARAM + }; + +**3. Register the property in ``class_init()``:** + +.. code-block:: c + + g_object_class_install_property (gobject_class, PROP_MODEL_SPECIFIC_PARAM, + g_param_spec_uint ("model-specific-param", "Model Specific Param", + "Custom parameter for new model architecture", + 0, 255, 0, /* min, max, default */ + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + +**4. Add getter/setter in ``set_property()`` and ``get_property()``:** + +.. code-block:: c + + static void + gst_dsp_kernel_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) + { + GstDspKernel *kernel = GST_DSP_KERNEL (object); + + switch (prop_id) { + /* ... existing properties ... */ + case PROP_MODEL_SPECIFIC_PARAM: + kernel->model_specific_param = g_value_get_uint (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + } + + static void + gst_dsp_kernel_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) + { + GstDspKernel *kernel = GST_DSP_KERNEL (object); + + switch (prop_id) { + /* ... existing properties ... */ + case PROP_MODEL_SPECIFIC_PARAM: + g_value_set_uint (value, kernel->model_specific_param); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + } + +**5. Verify the property is registered (after rebuild and install):** + +After rebuilding and installing the plugin, verify that the new property appears in the +element's property list using ``gst-inspect-1.0``: + +.. code-block:: bash + + root@am62dxx-evm:~# gst-inspect-1.0 tidspkernel | grep -A 5 "model-specific-param" + +Expected output: + +.. code-block:: text + + model-specific-param : Custom parameter for new model architecture + flags: readable, writable + Unsigned Integer. Range: 0 - 255 Default: 0 + +If your new property appears in the output with the correct type, range, and default value, +the registration was successful. You can now use it in GStreamer pipelines: + +.. code-block:: bash + + gst-launch-1.0 -v ... tidspkernel name=stft ... model-specific-param=1 ... ! ... + +Adding a New Property to ``titvm`` +----------------------------------- + +Similarly, for the TVM inference element, if you need a new parameter (for example, +``inference-threshold`` for filtering low-confidence predictions): + +**1. Header: ``gsttitvm.h`` - Add to struct:** + +.. code-block:: c + + struct _GstTiTvm + { + /* ... existing properties ... */ + gchar *model_path; + gchar *class_map_path; + guint top_k; + guint daemon_timeout_ms; + + /* NEW: Add your custom property */ + gfloat inference_threshold; /* Confidence threshold (0.0-1.0) */ + }; + +**2. Implementation: ``gsttitvm.cpp`` - Add property enum and register:** + +.. code-block:: c + + enum + { + PROP_0, + PROP_MODEL_PATH, + PROP_CLASS_MAP_PATH, + PROP_TOP_K, + PROP_DAEMON_TIMEOUT_MS, + + /* NEW: Add your property ID */ + PROP_INFERENCE_THRESHOLD + }; + + /* In class_init() */ + g_object_class_install_property (gobject_class, PROP_INFERENCE_THRESHOLD, + g_param_spec_float ("inference-threshold", "Inference Threshold", + "Minimum confidence score to report predictions (0.0-1.0)", + 0.0f, 1.0f, 0.5f, /* min, max, default */ + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + +**3. Use in inference post-processing:** + +.. code-block:: c + + /* After getting inference output from TVM, filter by threshold */ + for (guint i = 0; i < output_size; ++i) { + gfloat confidence = output_buffer[i]; + + if (confidence >= tvm->inference_threshold) { + GST_DEBUG_OBJECT (tvm, "Prediction %u: confidence=%.3f (above threshold %.3f)", + i, confidence, tvm->inference_threshold); + } + } + +Automatic Model Detection from Path +------------------------------------- + +If you want the plugins to auto-detect your new model based on its directory path name, +add a mapping in the model detection logic in ``gstdspkernel.cpp``: + +.. code-block:: c + + struct { + const gchar *path_substring; + guint selected_model; + guint model_elems; + guint fft_size; + guint hop_size; + guint window_frames; + } known_models[] = { + { "gcrn", 2, 322, 320, 160, 401 }, + { "gtcrn", 1, 322, 320, 160, 401 }, + { "dccrn", 0, 322, 320, 160, 401 }, + { "yamnet", 4, 64, 512, 160, 96 }, + { "vggish", 3, 128, 512, 160, 96 }, + + /* NEW: Add your model */ + { "my_custom_model", 5, 256, 512, 256, 100 }, + + { NULL, 0, 0, 0, 0, 0 } /* Sentinel */ + }; + +The ``gst_dsp_kernel_detect_model()`` function iterates this table and auto-sets +``selected-model``, ``model-elems``, ``fft-size``, ``hop-size``, and ``window-frames`` +when ``model-path`` contains the matching substring. + +.. _pub_edgeai_am62dx_building_plugin: + +Building the Modified Plugins +------------------------------ + +Plugin source files are located in the ``/opt/edgeai-gst-plugins/`` repository. + +**Source File Locations:** + +- ``ext/ti/gstdspkernel.h`` — Header for DSP kernel element (STFT/ISTFT/deinterleave) +- ``ext/ti/gstdspkernel.cpp`` — Implementation of DSP kernel element +- ``ext/ti/gsttitvm.h`` — Header for TVM inference element +- ``ext/ti/gsttitvm.cpp`` — Implementation of TVM inference element +- ``utility/gsttirpmsgctx.h`` / ``utility/gsttirpmsgctx.cpp`` — RPMsg context utilities + +After modifying any of these files, rebuild the plugin. The meson build system automatically +detects all source files in ``ext/ti/`` — **no meson.build file changes are required**. + +Native Build on the Board +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + root@am62dxx-evm:/opt/edgeai-gst-plugins# export SOC=am62d + root@am62dxx-evm:/opt/edgeai-gst-plugins# meson setup build --prefix=/usr \ + -Dpkg_config_path=pkgconfig \ + -Dtvm-plugins=enabled \ + -Dstft-plugins=enabled \ + -Ddl-plugins=disabled \ + -Denable-tidl=disabled + root@am62dxx-evm:/opt/edgeai-gst-plugins# ninja -C build + root@am62dxx-evm:/opt/edgeai-gst-plugins# sudo ninja -C build install + root@am62dxx-evm:/opt/edgeai-gst-plugins# sudo ldconfig + +The updated plugin will be installed to ``/usr/lib/gstreamer-1.0/libgstti.so``. diff --git a/source/devices/AM62DX/edgeai/linux_audio_inference_app.rst b/source/devices/AM62DX/edgeai/linux_audio_inference_app.rst new file mode 100644 index 000000000..d41d8339c --- /dev/null +++ b/source/devices/AM62DX/edgeai/linux_audio_inference_app.rst @@ -0,0 +1,365 @@ +.. _am62dx_linux_audio_inference_app: + +######################### +C++ Inference Application +######################### + +Introduction +============ + +The AM62DX audio inference application (``rpmsg_inference_example``) is a C++ application +for running configurable audio and tensor pipelines across Linux and the C7x™ DSP. It is part +of the `rpmsg-dma `__ repository. The +application enables: + +- **Real-time Audio Processing** — Process live input or WAV file inputs +- **DSP Offload** — Offload signal chains (ex: STFT, ISTFT and deinterleave/interleave) to the C7x™ DSP over RPMsg-DMA +- **TVM Inference** — Run TVM-compiled deep learning models on the C7x™ NPU +- **Flexible I/O** — WAV files, live ALSA capture, raw PCM streams, or binary tensors +- **JSON Pipeline Configuration** — Define processing chains via JSON files +- **Model Daemon Mode** — A model daemon hot-reloads artifacts without an application rebuild + +Supported Pipelines +=================== + +.. list-table:: + :header-rows: 1 + :widths: 30 30 20 20 + + * - Pipeline + - JSON file + - Default input + - Output + * - Speech enhancement (GCRN) + - ``pipeline_speech_enhancement.json`` + - ``input_audio.wav`` + - Enhanced WAV file + * - Audio classification (YAMNet) + - ``pipeline_audio_classification_yamnet.json`` + - ``speech_whistling2.wav`` + - Top-k class names and scores + * - Audio classification (VGGish) + - ``pipeline_audio_classification_vggish.json`` + - ``urbansound_16k.wav`` + - Top-k class names and scores + +Prerequisites +============= + +All runtime libraries, model artifacts, sample audio files, model daemon, and DSP firmware +verification are pre-packaged in the AM62DX EdgeAI image. See +:ref:`Models, Artifacts and Setup ` for details. + +To build the application from source, the following host tools are required: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Tool + - Purpose + * - ARM Clang Toolchain + - From Processor SDK Linux devkit; for cross-compilation + * - CMake 3.10 or higher + - Build system + * - ``make``, ``git`` + - Build utilities + * - Apache TVM headers (``TVM_ROOT``) + - Refer to `TI TVM Documentation `__ + +********************* +Building on AM62D EVM +********************* + +The application is pre-installed on the AM62DX EdgeAI image. If you need to rebuild it +natively on the AM62D EVM, clone the source and build: + +.. code-block:: console + + root@am62dxx-evm:~# git clone https://github.com/TexasInstruments/rpmsg-dma.git -b main + root@am62dxx-evm:~# cd rpmsg-dma + +The example is under ``example/edge-ai/`` and contains: + +.. code-block:: text + + example/edge-ai/ + ├── CMakeLists.txt + ├── src/ application source files + ├── include/ header files + ├── json_files/ pipeline JSON configurations + ├── labels/ classification label files + ├── input_audio/ sample WAV input files + ├── tvm-model-daemon.service + └── tvm-model-preload.service + +.. code-block:: console + + root@am62dxx-evm:~/rpmsg-dma# cmake -S . -B build -DBUILD_EDGE_AI_EXAMPLE=ON \ + -DBUILD_AUDIO_OFFLOAD_EXAMPLE=OFF \ + -DBUILD_2DFFT_OFFLOAD_EXAMPLE=OFF \ + -DBUILD_SIGNAL_CHAIN_BIQUAD_EXAMPLE=OFF + root@am62dxx-evm:~/rpmsg-dma# cmake --build build + +**************** +Building on Host +**************** + +#. Clone the source repository: + + .. code-block:: console + + host# git clone https://github.com/TexasInstruments/rpmsg-dma.git -b main + host# cd rpmsg-dma + +#. Download and install the AM62DX Linux SDK from |__SDK_DOWNLOAD_URL__|. + Extract the SDK tarball and run the installer script on your development host. + +#. Prepare the environment for cross compilation: + + .. code-block:: console + + host# source /linux-devkit/environment-setup + +#. Set the Neo-TVM path: + + .. code-block:: console + + host# export NEO_TVM_PATH=/path/to/neo-tvm + +#. Build only the edge-ai example: + + .. code-block:: console + + [linux-devkit]:> cmake -S . -B build -DBUILD_EDGE_AI_EXAMPLE=ON \ + -DBUILD_AUDIO_OFFLOAD_EXAMPLE=OFF \ + -DBUILD_2DFFT_OFFLOAD_EXAMPLE=OFF \ + -DBUILD_SIGNAL_CHAIN_BIQUAD_EXAMPLE=OFF + [linux-devkit]:> cmake --build build + +#. The following binaries are generated after a successful build: + + - ``build/example/edge-ai/rpmsg_inference_example`` — main inference application + - ``build/example/edge-ai/tvm_model_daemon`` — persistent model daemon + + Transfer the binaries to the AM62D EVM: + + .. code-block:: console + + host# scp build/example/edge-ai/rpmsg_inference_example root@:/usr/bin/ + host# scp build/example/edge-ai/tvm_model_daemon root@:/usr/bin/ + + Or copy directly to a mounted SD card: + + .. code-block:: console + + host# cp build/example/edge-ai/rpmsg_inference_example /mnt/rootfs/usr/bin/ + host# cp build/example/edge-ai/tvm_model_daemon /mnt/rootfs/usr/bin/ + +Basic Syntax +============ + +.. code-block:: console + + rpmsg_inference_example # file input from JSON + rpmsg_inference_example --device # live ALSA capture + rpmsg_inference_example --input-file # override input file + rpmsg_inference_example --stream # raw PCM from stdin + rpmsg_inference_example --debug # enable per-batch logs + rpmsg_inference_example --preload # preload default model + rpmsg_inference_example --version + rpmsg_inference_example --help + +***************************** +GCRN based Speech Enhancement +***************************** + +Speech enhancement removes background noise from input audio source while keeping the speech +intelligible — useful for voice calls, meeting recordings, hearables, and any product +that captures speech in a noisy environment. + +Speech Enhancement Pipeline using GCRN +-------------------------------------- + +Run with the default WAV file input: + +.. code-block:: console + + am62dxx-evm:~# rpmsg_inference_example /usr/share/tvm_inference/json/pipeline_speech_enhancement.json + edge-ai v0.0.4 (built Sep 21 2026 10:14:19) + =========================================== + RPMsg Inference Example + =========================================== + + [App] Pipeline type: speech_enhancement + [App] Description: GCRN 3-stage speech pipeline: STFT analyze -> TVM inference -> ISTFT synthesize + [App] TVM artifacts configured: /usr/share/tvm_inference/artifacts/gcrn + [App] Model changed (/usr/share/tvm_inference/artifacts/yamnet -> /usr/share/tvm_inference/artifacts/gcrn), requesting daemon reload... + [App] Daemon reloaded model: /usr/share/tvm_inference/artifacts/gcrn + [App] Input configured: /usr/share/tvm_inference/input/input_audio.wav + [App] Stages: 5 + [App] STFT parameters: HOP=160 MODEL_ELEMS=322 BATCH_N=64 TOTAL_FRAMES=401 + [App] ISTFT parameters: HOP=160 MODEL_ELEMS=322 BATCH_N=64 TOTAL_FRAMES=401 + [App] STFT spectral buffer: 516488 bytes + [App] ISTFT spectral buffer: 516488 bytes + [App] Audio file info: 156302 samples, 16000Hz, 1 channel(s) + [App] Loaded 156302 audio samples (9.76887 seconds) + eptdev = rpmsg-char-8-1303 + Created endpt device rpmsg-char-8-1303, fd = 4 port = 1027 + [App] DMA buffers (reused across sequential stages): + [App] A (PCM / planar): phys=0xc0600000 size=516488 + [App] B (interleaved): phys=0xc0680000 size=516488 + [App] Overlap-save chunking: + [App] TOTAL_FRAMES=401 OVERLAP_FRAMES=100 HOP_FRAMES=301 n_chunks=3 + [TVM] Connected to model daemon at /var/run/tvm-inference.sock + [App] TVM initialized, input shape [1,2,401,161] + [App] Chunk 1/3 [keep samples 0..56160] | STFT=17.3ms TVM=6782.4ms ISTFT=18.4ms total=6821.0ms + [App] Chunk 2/3 [keep samples 8000..56160] | STFT=16.3ms TVM=6781.0ms ISTFT=17.7ms total=6817.6ms + [App] Chunk 3/3 [keep samples 8000..59982] | STFT=16.3ms TVM=6780.0ms ISTFT=18.2ms total=6817.3ms + [App] All chunks done | total=20456.2ms | 976 output frames (156302 samples) + [App] Saving audio to: processed_output.wav + [App] Output file info: 156302 samples, 16000Hz, 1 channel(s) + [App] Successfully saved 156302 samples to processed_output.wav + [App] Duration: 9.8 seconds + [App] Saved to processed_output.wav + PASSED + +****************************************** +YAMNet & VGGish based Audio Classification +****************************************** + +Audio classification identifies *what* sound is present in input audio source — a siren, +glass breaking, a dog barking, a machine running. YAMNet recognizes 521 general sound +classes, while VGGish is a smaller 10-class model. Pick YAMNet for broad coverage, or +VGGish when you need a lighter model for a fixed set of sounds. + +Audio Classification pipeline using YAMNet +------------------------------------------ + +Run with the default WAV file input: + +.. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example /usr/share/tvm_inference/json/pipeline_audio_classification_yamnet.json + edge-ai v0.0.4 (built Sep 21 2026 10:14:19) + =========================================== + RPMsg Inference Example + =========================================== + + [App] Pipeline type: audio_classification + [App] Description: YAMNet: speech -> STFT log-mel spectrogram -> TVM classification (521 classes) + [App] TVM artifacts configured: /usr/share/tvm_inference/artifacts/yamnet + [App] Model changed (/usr/share/tvm_inference/artifacts/vggish -> /usr/share/tvm_inference/artifacts/yamnet), requesting daemon reload... + [App] Daemon reloaded model: /usr/share/tvm_inference/artifacts/yamnet + [App] Input configured: /usr/share/tvm_inference/input/speech_whistling2.wav + [App] Stages: 2 + [App] STFT parameters: HOP=160 MODEL_ELEMS=64 BATCH_N=64 TOTAL_FRAMES=96 + [TVM] Connected to model daemon at /var/run/tvm-inference.sock + [App] TVM initialized, input shape [1,1,96,64] + [App] Loaded 521 class labels from /usr/share/tvm_inference/labels/yamnet_label_list.txt + [App] Audio file info: 78720 samples, 16000Hz, 1 channel(s) + [App] Loaded 78720 audio samples (4.92 seconds) + eptdev = rpmsg-char-8-1298 + Created endpt device rpmsg-char-8-1298, fd = 5 port = 1027 + [App] DMA buffers: + [App] audio in: phys=0xc01b8000 size=20480 + [App] mel out: phys=0xc01d8000 size=24576 + [App] Total frames: 492 | Full chunks: 5 + [App] Chunk 1/5 STFT=21.9ms TVM=26.1ms total=48.0ms + [App] Chunk 1 top-5: + [App] 1. 4.7947 Speech + [App] 2. -2.6637 Narration, monologue + [App] 3. -3.7292 Speech synthesizer + [App] 4. -3.7292 Conversation + [App] 5. -4.7947 Inside, small room + [App] Chunk 2/5 STFT=9.4ms TVM=25.8ms total=35.2ms + [App] Chunk 2 top-5: + [App] 1. -1.5982 Animal + [App] 2. -1.5982 Bird + [App] 3. -1.5982 Whistling + [App] 4. -2.1310 Wild animals + [App] 5. -2.6637 Domestic animals, pets + [App] Chunk 3/5 STFT=9.4ms TVM=26.0ms total=35.4ms + [App] Chunk 3 top-5: + [App] 1. 3.1965 Whistling + [App] 2. -3.1965 Whistle + [App] 3. -3.7292 Alarm + [App] 4. -5.3275 Inside, small room + [App] 5. -5.8602 Bird + [App] Chunk 4/5 STFT=9.4ms TVM=26.3ms total=35.7ms + [App] Chunk 4 top-5: + [App] 1. 0.0000 Whistling + [App] 2. -1.0655 Whistle + [App] 3. -1.5982 Alarm + [App] 4. -5.8602 Inside, small room + [App] 5. -6.9257 Water + [App] Chunk 5/5 STFT=9.4ms TVM=26.0ms total=35.4ms + [App] Chunk 5 top-5: + [App] 1. 2.6637 Whistle + [App] 2. 1.0655 Alarm + [App] 3. 0.5327 Whistling + [App] 4. -5.3275 Inside, small room + [App] 5. -5.8602 Steam whistle + [App] Done: 492 mel frames | 5 TVM chunks | 190.3ms + PASSED + [App] Application exited with code 0 + +Live ALSA device input: + +.. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example \ + /usr/share/tvm_inference/json/pipeline_audio_classification_yamnet.json \ + --device plughw:0,0 + +Stream raw PCM from stdin (mono, S16_LE, 16 kHz): + +.. code-block:: console + + root@am62dxx-evm:~# arecord -t raw -f S16_LE -c 1 -r 16000 | \ + rpmsg_inference_example \ + /usr/share/tvm_inference/json/pipeline_audio_classification_yamnet.json \ + --stream + +Audio Classification Pipeline using VGGish +------------------------------------------ + +Run with the default WAV file input: + +.. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example /usr/share/tvm_inference/json/pipeline_audio_classification_vggish.json + edge-ai v0.0.4 (built Sep 21 2026 10:14:19) + =========================================== + RPMsg Inference Example + =========================================== + + [App] Pipeline type: audio_classification + [App] Description: VGGish: audio -> STFT log-mel spectrogram -> TVM classification (UrbanSound8K 10 classes) + [App] TVM artifacts configured: /usr/share/tvm_inference/artifacts/vggish + [App] Model changed (/usr/share/tvm_inference/artifacts/gcrn -> /usr/share/tvm_inference/artifacts/vggish), requesting daemon reload... + [App] Daemon reloaded model: /usr/share/tvm_inference/artifacts/vggish + [App] Input configured: /usr/share/tvm_inference/input/urbansound_16k.wav + [App] Stages: 2 + [App] STFT parameters: HOP=512 MODEL_ELEMS=64 BATCH_N=64 TOTAL_FRAMES=126 + [TVM] Connected to model daemon at /var/run/tvm-inference.sock + [App] TVM initialized, input shape [1,1,126,64] + [App] Loaded 10 class labels from /usr/share/tvm_inference/labels/vggish_label_list.txt + [App] Audio file info: 128000 samples, 16000Hz, 1 channel(s) + [App] Loaded 128000 audio samples (8 seconds) + eptdev = rpmsg-char-8-1290 + Created endpt device rpmsg-char-8-1290, fd = 5 port = 1027 + [App] DMA buffers: + [App] audio in: phys=0xc02b0000 size=65536 + [App] mel out: phys=0xc01b8000 size=32256 + [App] Total frames: 250 | Full chunks: 1 + [App] Chunk 1/1 STFT=44.4ms TVM=31.1ms total=75.5ms + [App] Chunk 1 top-5: + [App] 1. 8.4112 Street music + [App] 2. 8.2060 Children playing + [App] 3. -4.5133 Air conditioner + [App] 4. -5.1288 Dog bark + [App] 5. -5.5391 Car horn + [App] Done: 250 mel frames | 1 TVM chunks | 75.6ms + PASSED + [App] Application exited with code 0 diff --git a/source/devices/AM62DX/edgeai/model_compile_evaluate.rst b/source/devices/AM62DX/edgeai/model_compile_evaluate.rst new file mode 100644 index 000000000..817ba0a58 --- /dev/null +++ b/source/devices/AM62DX/edgeai/model_compile_evaluate.rst @@ -0,0 +1,251 @@ +.. include:: + +.. _edgeai-compile-evaluate: + +########################## +Model Compile and Evaluate +########################## + +The :ref:`EdgeAI Model Zoo ` runs *pre-compiled* artifacts +on the |__PART_FAMILY_NAME__| target. This section covers the **x86 host +workflow** that produces those artifacts: how to compile an audio model for +the C7\ |trade| NPU, run host-emulation inference, and measure accuracy against a +dataset - using **edgeai-tidlrunner**. It runs entirely on an Ubuntu x86 PC; +nothing here touches the target. + +This is the workflow for *any* audio model, not just the reference models. The +four reference models below are worked examples. To bring your own model you +point the same ``tidlrunner-cli`` at your ONNX graph and a per-model YAML config, +using a reference config as a starting template. Which operators offload to the +C7\ |trade| NPU (vs. fall back to the Arm core) is determined by TIDL - see +`edgeai-tidl-tools `__ for +the authoritative list of supported operators. + +***************** +edgeai-tidlrunner +***************** + +edgeai-tidlrunner is a bring-your-own-model (BYOM) command-line wrapper over the +core `edgeai-tidl-tools `__ +compilation stack. A single YAML config per model drives every stage, and one +CLI - ``tidlrunner-cli`` - exposes the stages this section uses: + +.. list-table:: + :header-rows: 1 + :widths: 18 82 + + * - Command + - What it does + * - ``compile`` + - Imports the ONNX graph, quantizes and partitions it, and writes the + deployable TIDL/TVM-RT artifacts. + * - ``infer`` + - Runs the compiled model on the x86 host (PC emulation) to check output + correctness before deploying to the target. + * - ``evaluate`` + - Runs the compiled model over a dataset and reports accuracy or + speech-enhancement metrics. + * - ``inspect`` + - One-shot compile + inference + reference comparison that emits an + interactive HTML analysis report (see :ref:`Model Inspect `). + +All commands are run from the repository root (``edgeai-tidlrunner/``) and take +``--config_path `` pointing at the per-model YAML. The same config file is +reused across ``compile``, ``infer``, and ``evaluate``. + +.. _edgeai-am62d-setup: + +***** +Setup +***** + +Model compilation runs on an x86 PC (Ubuntu Linux recommended). The AM62D audio +flow is self-contained: a single setup script installs the whole TVM toolchain +into a dedicated Python 3.10 environment. The TIDL tools target Python 3.10. +These steps assume `pyenv `__. + +#. **Clone the repository** (https://github.com/TexasInstruments/edgeai-tidlrunner): + + .. code-block:: console + + $ git clone https://github.com/TexasInstruments/edgeai-tidlrunner.git + $ cd edgeai-tidlrunner + +#. **Create and activate a dedicated Python 3.10 environment.** The AM62D flow + uses a TI TVM wheel; keep it in its own venv (``tidlrunner-am62d``) + so it never clobbers a standard ``tidlrunner`` setup: + + .. code-block:: console + + $ pyenv install 3.10 + $ pyenv virtualenv 3.10 tidlrunner-am62d + $ pyenv activate tidlrunner-am62d + +#. **Run the AM62D setup script.** With the venv active, one script installs + everything this flow needs: + + .. code-block:: console + + $ ./devices/setup_am62d.sh + + It downloads the ARM GCC 15.2 and C7000 CGT 5.0.0.LTS cross-toolchains into + ``tools/tidl_tools_package/bin/`` (skipped if already present), installs the + x86 TI TVM wheel - which bundles the AM62D x86 TIDL tools *inside* + the package - and installs ``tidlrunner[pc,audio]``, ``tools``, ``onnxruntime``, + and ``tidl_onnx_model_optimizer``. The ``[audio]`` extra pulls in the audio + packages (``librosa``, ``soundfile``, ``scipy``, ``pesq``, ``pystoi``, + ``scikit-learn``). No separate ``pip install`` is needed. + +#. **Source the env script once per shell.** TVM ``dlopen``\ s the TIDL runtime + (``libvx_tidl_rt.so``) before Python's own import runs, so the tool paths must be + live in the shell *before* ``tidlrunner-cli`` starts. Source (do **not** execute) + ``devices/am62d_env.sh`` once; every ``tidlrunner-cli`` command run afterward in + that shell picks the vars up: + + .. code-block:: console + + $ source devices/am62d_env.sh + + It exports: + + .. list-table:: + :header-rows: 1 + :widths: 26 74 + + * - Variable + - Value + * - ``TIDL_TOOLS_PATH`` + - ``/3rdparty/x86_tidl_tools/AM62D`` (resolved from the + active venv's installed ``tvm``) + * - ``LD_LIBRARY_PATH`` + - prepends ``TIDL_TOOLS_PATH`` so ``libvx_tidl_rt.so`` resolves + * - ``ARM64_GCC_PATH`` + - the aarch64 GNU toolchain installed by ``setup_am62d.sh`` under + ``tools/tidl_tools_package/bin/`` (override by exporting before sourcing) + * - ``CGT7X_ROOT`` + - the C7000 code-generation tools installed by ``setup_am62d.sh`` under + ``tools/tidl_tools_package/bin/`` (override by exporting before sourcing) + * - ``SOC`` + - ``am62d`` + +Datasets +======== + +Accuracy evaluation needs the reference datasets. Compilation alone can run on +random data (for latency-only measurement), but ``evaluate`` requires the real +sets. Download them from the repository root: + +.. code-block:: console + + $ # UrbanSound8K (~5.6 GB) - sound classification (VGGish11, YAMNet) + $ bash examples/audio/scripts/download_urbansound8k.sh + + $ # VoiceBank-DEMAND-16k (~2 GB) - speech enhancement (GCRN, GTCRN) + $ python3 examples/audio/scripts/download_voicebank_demand.py + +They land under ``data/datasets/UrbanSound8K/`` and +``data/datasets/VoiceBank-DEMAND-16k/`` - the paths the model configs expect. +The classification configs use UrbanSound8K **fold 10** (837 samples) as the +test set; the speech-enhancement configs use the VoiceBank-DEMAND-16k **test** set +(824 files). + +The reference ONNX models auto-download at compile time via ``.link`` files, so +fetching them ahead of time is **optional**. To pre-download all reference ONNX models: + +.. code-block:: console + + $ bash examples/audio/scripts/download_audio_models.sh + +******************** +Compile and evaluate +******************** + +Each model has one config that drives all stages. All four run under the +``tidlrunner-am62d`` environment with ``devices/am62d_env.sh`` already sourced +(see `Setup`_). Three models compile to the C7\ |trade| NPU through the TVM runtime; +GTCRN runs ARM-only through ONNX Runtime (no C7\ |trade| NPU offload) but uses the +same environment. The AM62D config paths are: + +.. list-table:: + :header-rows: 1 + :widths: 12 10 12 18 54 + + * - Model + - Model ID + - Runtime + - Tensor Bits + - Config path (relative to ``edgeai-tidlrunner/``) + * - VGGish11 + - ``ac-0101`` + - TVM-RT + - 8 + - ``data/configs/samples/models/audio/audio_classification/urbansound8k/vggish11_tvmrt_config.yaml`` + * - YAMNet + - ``ac-0201`` + - TVM-RT + - 8 + - ``data/configs/samples/models/audio/audio_classification/urbansound8k/yamnet_tvmrt_config.yaml`` + * - GCRN + - ``ase-0201`` + - TVM-RT + - 16 (LSTM), FP32 (Enc/Dec) + - ``data/configs/samples/models/audio/speech_enhancement/voicebank_demand_16k/gcrn_fixed_4sec_tvmrt_config.yaml`` + * - GTCRN + - ``ase-0100`` + - ONNX-RT + - FP32 + - ``data/configs/samples/models/audio/speech_enhancement/voicebank_demand_16k/gtcrn_dns3_config.yaml`` + +VGGish11 (TVM-RT, 8-bit) +======================== + +Runs on the C7\ |trade| NPU (``tidl_offload: true``): + +.. code-block:: console + + $ CFG=data/configs/samples/models/audio/audio_classification/urbansound8k/vggish11_tvmrt_config.yaml + $ tidlrunner-cli compile --config_path $CFG + $ tidlrunner-cli infer --config_path $CFG + $ tidlrunner-cli evaluate --config_path $CFG + +``evaluate`` reports top-1 / top-5 / macro-F1 over UrbanSound8K fold 10. + +YAMNet (TVM-RT, 8-bit) +====================== + +.. code-block:: console + + $ CFG=data/configs/samples/models/audio/audio_classification/urbansound8k/yamnet_tvmrt_config.yaml + $ tidlrunner-cli compile --config_path $CFG + $ tidlrunner-cli infer --config_path $CFG + $ tidlrunner-cli evaluate --config_path $CFG + +GCRN (TVM-RT, 16-bit) +===================== + +GCRN is C7\ |trade| NPU-offloaded at 16-bit (``tidl_offload: true``, fixed 4-second +input): + +.. code-block:: console + + $ CFG=data/configs/samples/models/audio/speech_enhancement/voicebank_demand_16k/gcrn_fixed_4sec_tvmrt_config.yaml + $ tidlrunner-cli compile --config_path $CFG + $ tidlrunner-cli infer --config_path $CFG + $ tidlrunner-cli evaluate --config_path $CFG + +``evaluate`` reports PESQ / STOI / SI-SDR over the VoiceBank-DEMAND-16k test set. + +GTCRN (ONNX-RT, ARM-only) +========================= + +GTCRN has a dynamic time axis and TIDL-unsupported operators, so it runs on the +Arm Cortex-A cores through ONNX Runtime (``tidl_offload: false``) - no C7\ |trade| NPU +offload. It uses the same ``tidlrunner-am62d`` environment as the other models; +it simply does not offload to the DSP: + +.. code-block:: console + + $ CFG=data/configs/samples/models/audio/speech_enhancement/voicebank_demand_16k/gtcrn_dns3_config.yaml + $ tidlrunner-cli compile --config_path $CFG + $ tidlrunner-cli infer --config_path $CFG + $ tidlrunner-cli evaluate --config_path $CFG diff --git a/source/devices/AM62DX/edgeai/model_inspect.rst b/source/devices/AM62DX/edgeai/model_inspect.rst new file mode 100644 index 000000000..d8b0f5b5c --- /dev/null +++ b/source/devices/AM62DX/edgeai/model_inspect.rst @@ -0,0 +1,65 @@ +.. _edgeai-model-inspect: + +############# +Model Inspect +############# + +Model Inspector analyzes a model on the x86 host after (or instead of) the +:ref:`compile and evaluate ` flow. It uses the same +edgeai-tidlrunner installation and environment described in +:ref:`Setup `. + +*************** +Model Inspector +*************** + +``tidlrunner-cli inspect`` is a one-shot analysis pipeline: it compiles the +model for TIDL, runs inference, re-runs a non-TIDL reference for comparison, +computes per-layer signal-to-noise ratios, and emits a **self-contained +interactive HTML report** - no need to run ``compile`` and ``infer`` +separately. The report opens in any browser and lands at: + +.. code-block:: text + + work_dirs/compile////inspect/modelinspector.html + +```` is the model's work directory, named after its model ID, +runtime, task, dataset, and model, e.g. +``ase-0201_tvmrt_speech_enhancement_voicebank_demand_16k_gcrn_fixed_4sec_onnx``. + +The report has three tabs: + +- **Model Summary** - acceleration ratio (layers on the C7x™ DSP vs. Arm + fallback), input/output specs, and an interactive ONNX graph. +- **Subgraphs** - the TIDL subgraph(s) offloaded to the DSP, with per-layer + identity and a quantization analysis (INT8 vs. FP32 activation histograms and + scatter plots). +- **Performance** - *estimated* per-layer processing time, DSP cycle breakdown + (total / IO / kernel), memory usage (DDR / L2 / MSMC), and SNR metrics. + +.. note:: + + All Performance-tab numbers are **estimates** from the compiler, not + on-target measurements. Model Inspector requires TIDL tools **11.02 or + later** (earlier versions emit SVG subgraph files it cannot read). + +GCRN showcase - an offloaded DSP subgraph +========================================= + +Because GCRN now compiles with ``tidl_offload: true`` on AM62D, its inspector +report shows a **real C7x™ DSP subgraph** rather than an all-Arm fallback: +the Subgraphs tab lists a single offloaded subgraph (16-bit) with its layer +mapping, and the Performance tab breaks that subgraph down by cycles and memory. +Generate it with the AM62D env sourced (see :ref:`Setup `): + +.. code-block:: console + + $ CFG=data/configs/samples/models/audio/speech_enhancement/voicebank_demand_16k/gcrn_fixed_4sec_tvmrt_config.yaml + $ tidlrunner-cli inspect --config_path $CFG + +For contrast, running ``inspect`` on the GTCRN (ONNX-RT) config produces a +report with **no** DSP subgraph - every layer is Arm fallback - which is what a +non-offloaded model looks like in the inspector. GTCRN also runs with +``tidl_offload: false``, so its work directory uses a ``notidl`` segment in +place of the ```` shown in the path template above, e.g. +``work_dirs/compile/AM62D/notidl//...``. diff --git a/source/devices/AM62DX/edgeai/model_zoo.rst b/source/devices/AM62DX/edgeai/model_zoo.rst new file mode 100644 index 000000000..41df86f0c --- /dev/null +++ b/source/devices/AM62DX/edgeai/model_zoo.rst @@ -0,0 +1,314 @@ +.. include:: + +.. _edgeai-model-zoo: + +################## +EdgeAI Model Zoo +################## + +The EdgeAI ModelZoo delivers out-of-box audio deep-learning demos that run +directly on the |__PART_FAMILY_NAME__| target. Each reference model ships as an +`ONNX `__ graph plus, where applicable, a pre-compiled +TVM-RT + TIDL artifact, so you can reproduce these examples on the AM62D without +compiling anything yourself. Two demo front-ends are provided: standalone +Python inference scripts and Jupyter notebooks. The same on-target runtime hosts +your own compiled artifacts once you build them through the x86 flow. This +section runs entirely **on the target**; the x86 compile/evaluate flow is covered +in :ref:`Model Compile and Evaluate `. + +.. note:: + + The models in this repository are made available for experimentation and + development. They are not intended for deployment in production. + +********** +What ships +********** + +.. list-table:: + :header-rows: 1 + :widths: 18 26 14 14 28 + + * - Model + - Task + - Model ID + - On target + - Demo + * - VGGish11 + - Sound classification (UrbanSound8K) + - ``ac-0101`` + - C7\ |trade| NPU (TVM-RT + TIDL, 8-bit) + - Script + notebook + * - YAMNet + - Sound classification (UrbanSound8K) + - ``ac-0201`` + - C7\ |trade| NPU (TVM-RT + TIDL, 8-bit) + - Script + notebook + * - GCRN + - Speech enhancement (VoiceBank-DEMAND-16k) + - ``ase-0201`` + - C7\ |trade| NPU (TVM-RT + TIDL, 16-bit) + - Script + notebook + * - GTCRN + - Speech enhancement (VoiceBank-DEMAND-16k) + - ``ase-0100`` + - Arm core (ONNX runtime, FP32) + - Script + notebook + +VGGish11, YAMNet, and GCRN each ship a pre-compiled TVM-RT + TIDL artifact and +run offloaded to the C7\ |trade| NPU. GTCRN ships **no** compiled artifact - it +runs FP32 on the Arm core through the ONNX runtime, packaged as a plain ONNX +Runtime artifact. + +*************** +On-target setup +*************** + +The steps below run on the AM62D target (aarch64), on the Linux command line of +the Processor SDK rootfs. The notebooks and scripts run in a Python virtual +environment that reuses the TIDL-enabled TVM runtime already present in the +rootfs. + +#. **Clone the repository.** Clone the AudioAI ModelZoo repository + (https://github.com/TexasInstruments/audioai-modelzoo): + + .. code-block:: console + + $ mkdir -p ~/tidl && cd ~/tidl + $ git clone https://github.com/TexasInstruments/audioai-modelzoo.git + $ cd audioai-modelzoo + +#. **Download the model artifacts.** Downloads the three pre-compiled + TVM-RT + TIDL artifacts (VGGish11, YAMNet, GCRN) plus the GTCRN ONNX Runtime + artifact into ``model_artifacts//am62d/``: + + .. code-block:: console + + $ ./download_models.sh -y + + The script defaults to an interactive selection menu; the ``-y`` flag + downloads everything non-interactively. Add ``-l`` to print the source URLs + without downloading. + +#. **Create the virtual environment.** Creates the venv at + ~/venv/modelzoo with ``--system-site-packages`` so the TIDL-enabled + runtime from the rootfs is visible: + + .. code-block:: console + + $ ./setup_venv.sh + +#. **Activate the venv.** Activate it in every new shell before running the + demos: + + .. code-block:: console + + $ source ~/venv/modelzoo/bin/activate + +********************** +Inference script demos +********************** + +Run each script from its own model directory in the activated venv. All four +models ship a command-line script. + +YAMNet (sound classification) +============================= + +.. code-block:: console + + $ cd ~/tidl/audioai-modelzoo/inference/yamnet_sc + $ python3 yamnet_infer_audio.py --audio-file samples/miaow_16k.wav + +The script converts the audio into log-mel patches, runs each patch through +the TVM-RT + TIDL session, and averages the per-patch scores. For the bundled +``miaow_16k.wav`` clip the top classes are cat/animal sounds. Expected output +(abridged): + +.. code-block:: text + + ====================================================================== + YAMNET INFERENCE SUMMARY + ====================================================================== + + Audio File: miaow_16k.wav + Artifacts: ac-0201_tvmrt_audio_classification_urbansound8k_yamnet_onnx + Audio Duration: 6.73 seconds + Number of Patches: 7 + + ====================================================================== + TOP 10 PREDICTIONS (averaged across all patches): + ====================================================================== + Rank Score Class + ---------------------------------------------------------------------- + 1 ... Cat + 2 ... Animal + ... + ====================================================================== + PERFORMANCE METRICS: + ====================================================================== + Inference Time (all patches): 17.53 ms total | 7 patches + RTF = 0.0026 (0.018s / 6.73s) + ====================================================================== + +A second sample, ``samples/speech_whistling2.wav``, is also included. + +VGGish11 (sound classification) +=============================== + +.. code-block:: console + + $ cd ~/tidl/audioai-modelzoo/inference/vggish11_sc + $ python3 vggish_infer_audio.py --audio-file sample_wav/139951-9-0-9.wav + +The bundled clip is UrbanSound8K class 9 (street music). Expected +output (abridged): + +.. code-block:: text + + ====================================================================== + VGGISH11 INFERENCE SUMMARY + ====================================================================== + + Audio File: 139951-9-0-9.wav + Artifacts: ac-0101_tvmrt_audio_classification_urbansound8k_vggish11_onnx + Audio Duration: 4.00 seconds + Top Class: Street music (Class 9) - Score: ... + + Performance: + Inference: 8.88 ms + Throughput: RTF = 0.0022 (0.009s / 4.00s) + ====================================================================== + +GCRN (speech enhancement) +========================= + +.. code-block:: console + + $ cd ~/tidl/audioai-modelzoo/inference/gcrn_se + $ python3 gcrn_infer_audio.py --input sample_wav/noisy.wav + +The script writes the enhanced audio to ``enhanced.wav`` beside the input and a +per-block timing report to ``gcrn-benchmark.md``. Pass ``--output`` to change +the destination wav. Expected output (abridged): + +.. code-block:: text + + Machine: aarch64 SOC: am62d TIDL_VER: ... + Loading TVM + TIDL session... + Running inference on noisy.wav (9.77s)... + Wrote enhanced audio to sample_wav/enhanced.wav + + GCRN TVM+TIDL benchmark (3 chunk(s), 9.77s audio) + + block total (ms) per-chunk (ms) + ... + Real-Time Factor (inference / audio) = 3.9500 + + Wrote benchmark report to sample_wav/gcrn-benchmark.md + +GTCRN (speech enhancement) +========================== + +.. code-block:: console + + $ cd ~/tidl/audioai-modelzoo/inference/gtcrn_se + $ python3 gtcrn_infer_audio.py --input sample_wav/mix.wav + +Unlike the other three models, GTCRN has no TVM-RT + TIDL artifact - the script +loads the ONNX graph directly and runs it on ``CPUExecutionProvider`` (Arm +core, FP32). It writes the enhanced audio to ``enh_onnx.wav`` beside the input; +pass ``--output`` to change the destination wav. Expected output (abridged): + +.. code-block:: text + + Machine: aarch64 + Model: .../ase-0100_onnxrt_speech_enhancement_voicebank_demand_16k_gtcrn_dns3_onnx/model/gtcrn_dns3.onnx + Loading ONNX Runtime session... + Running inference on mix.wav (9.77s)... + Wrote enhanced audio to sample_wav/enh_onnx.wav + + GTCRN ONNX Runtime (CPU) benchmark (9.77s audio) + + block total (ms) + ------------------------ ---------- + pre-processing (STFT) ... + model inference ... + post-processing (iSTFT) ... + end-to-end ... + + Real-Time Factor (inference / audio) = 0.0700 + +GTCRN is already faster than real time on the Arm core, so it needs no NPU +offload. + +.. _edgeai-model-zoo-notebooks: + +********************** +Jupyter notebook demos +********************** + +All four reference models have a Jupyter notebook demo. Start Jupyter Lab from +the activated venv: + +.. code-block:: console + + $ cd ~/tidl/audioai-modelzoo + $ ./jupyter_lab_venv.sh + +The launcher prints a highlighted access URL. Open the URL in a browser on a +machine that can reach the target and enter the token ``tidl`` when prompted: + +.. code-block:: text + + Access URL: http://:8888/lab?token=tidl + +Jupyter Lab opens with the GCRN notebook loaded. To run a different model, pick +its notebook from the file browser in the left sidebar. The notebooks are: + +.. list-table:: + :header-rows: 1 + :widths: 22 40 + + * - Model + - Notebook + * - GCRN + - ``inference/gcrn_se/gcrn_inference.ipynb`` + * - GTCRN + - ``inference/gtcrn_se/gtcrn_inference.ipynb`` + * - VGGish11 + - ``inference/vggish11_sc/vggish_inference.ipynb`` + * - YAMNet + - ``inference/yamnet_sc/yamnet_inference.ipynb`` + +****************** +Compiled artifacts +****************** + +``download_models.sh`` places the artifacts under +``model_artifacts//am62d/``. Each model gets its own directory, +named after its model ID, containing a ``model/`` folder (the trained ONNX +model) and, where applicable, an ``artifacts/`` folder (the compiled model +artifacts). VGGish11, YAMNet, and GCRN each get a pre-compiled TVM-RT + TIDL +artifact: + +- ``ac-0101`` - ``ac-0101_tvmrt_audio_classification_urbansound8k_vggish11_onnx`` (VGGish11) +- ``ac-0201`` - ``ac-0201_tvmrt_audio_classification_urbansound8k_yamnet_onnx`` (YAMNet) +- ``ase-0201`` - ``ase-0201_tvmrt_speech_enhancement_voicebank_demand_16k_gcrn_fixed_4sec_onnx`` (GCRN) + +Each of those artifact directories' ``artifacts/`` folder holds the TVM-RT +deployment artifacts (``deploy_lib.so``, ``deploy_graph.json``, ``deploy_param.params``) +in both target (``.evm``, aarch64) and host (``.pc``, x86) variants. The demo +scripts and notebooks select the ``.evm`` variant automatically on the target. + +GTCRN (``ase-0100``) gets an ONNX Runtime artifact instead of a +compiled one: + +- ``ase-0100`` - ``ase-0100_onnxrt_speech_enhancement_voicebank_demand_16k_gtcrn_dns3_onnx`` (GTCRN) + +Its directory holds only the ``model/`` folder, with the ``gtcrn_dns3.onnx`` +graph - there is no ``artifacts/`` folder since no TIDL compilation is +involved. The demo script and notebook load the ONNX model directly with +``CPUExecutionProvider`` and run FP32 on the Arm core. Compiled TVM-RT + TIDL +artifacts for your own models are produced by the x86 workflow in +:ref:`Model Compile and Evaluate `. diff --git a/source/devices/AM62DX/edgeai/models_artifacts_and_setup.rst b/source/devices/AM62DX/edgeai/models_artifacts_and_setup.rst new file mode 100644 index 000000000..dab1c4e4e --- /dev/null +++ b/source/devices/AM62DX/edgeai/models_artifacts_and_setup.rst @@ -0,0 +1,180 @@ +.. _am62dx_edgeai_models_artifacts_and_setup: + +######################### +Model Artifacts and Setup +######################### + +The AM62DX EdgeAI image ships with three pre-compiled, pre-packaged models ready to run +out of the box — no compilation required. These models, artifacts, and setup steps are +common to both the Linux inference application and the GStreamer pipelines. + +******************** +Pre-Packaged Models +******************** + +.. list-table:: + :header-rows: 1 + :widths: 15 20 10 25 30 + + * - Model + - Type + - ModelId + - Artifacts path + - Description + * - GCRN + - Speech enhancement + - 2 + - ``/usr/share/tvm_inference/artifacts/gcrn/`` + - Removes background noise from speech; produces enhanced WAV output + * - YAMNet + - Audio classification + - 4 + - ``/usr/share/tvm_inference/artifacts/yamnet/`` + - 521-class AudioSet audio classification + * - VGGish + - Audio classification + - 3 + - ``/usr/share/tvm_inference/artifacts/vggish/`` + - 10-class UrbanSound8K audio classification + +*************** +Model Artifacts +*************** + +Each model directory under ``/usr/share/tvm_inference/artifacts//`` contains: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - File + - Description + * - ``deploy_graph.json`` + - Graph description; defines the neural network topology and input/output shapes + * - ``deploy_lib.so`` + - Compiled model library; shared object for TVM runtime + * - ``deploy_param.params`` + - Model weights; binary parameter data + +*********** +Label Files +*********** + +Classification models ship with label files under ``/usr/share/tvm_inference/labels/``: + +.. list-table:: + :header-rows: 1 + :widths: 35 65 + + * - File + - Description + * - ``yamnet_label_list.txt`` + - 521 lines; one AudioSet class name per line for YAMNet + * - ``vggish_label_list.txt`` + - 10 lines; one UrbanSound8K class name per line for VGGish + +*********** +Input Audio +*********** + +All pipelines expect **16 kHz, mono, signed 16-bit PCM WAV** input. Sample files in +this format are pre-installed under ``/usr/share/tvm_inference/input/``: + +.. list-table:: + :header-rows: 1 + :widths: 35 25 40 + + * - File + - Used with + - Description + * - ``input_audio.wav`` + - GCRN + - Sample noisy speech recording + * - ``speech_whistling2.wav`` + - YAMNet + - Sample audio for event classification + * - ``urbansound_16k.wav`` + - YAMNet, VGGish + - Sample urban sound recording + +If your source audio is in a different format, convert it before use: + +.. code-block:: bash + + ffmpeg -i input.wav -ar 16000 -ac 1 -sample_fmt s16 output_16k.wav + +Or convert inline inside a GStreamer pipeline by inserting ``audioconvert`` and +``audioresample`` before the ``audio/x-raw`` capsfilter. + +********************** +Pre-Packaged Libraries +********************** + +The following libraries are pre-installed on the AM62DX EdgeAI image. No installation +is required to run the out-of-box pipelines on the AM62D EVM. + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Library + - Description + * - ``libti_rpmsg_dma`` + - RPMsg and DMA client library for inter-processor communication + * - ``libtvm_runtime.so`` + - Apache TVM ARM64 runtime for model inference + * - ``libgstti.so`` + - GStreamer plugin library containing ``tidspkernel`` and ``titvm`` elements + * - ``libasound`` + - ALSA audio library + * - ``libsndfile`` + - Audio file I/O library + * - ``libjson-c`` + - JSON parsing library + +************ +Model Daemon +************ + +The ``tvm_model_daemon`` keeps a TVM model loaded in memory and serves inference +requests over a Unix domain socket at ``/var/run/tvm-inference.sock``. Both the Linux +inference application and the GStreamer ``titvm`` element connect to it automatically, +eliminating repeated artifact loading for each run. + +Start the daemon: + +.. code-block:: console + + root@am62dxx-evm:~# systemctl enable --now tvm-model-daemon.service + root@am62dxx-evm:~# journalctl -u tvm-model-daemon -f + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: rpmsg: Opened endpoint device /dev/rpmsg4 + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: [daemon] Loading TVM artifacts from: /usr/share/tvm_inference/artifacts/gcrn + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: [daemon] C7x firmware OK and already running + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: c7x: Connected to compute service + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: c7x: Staging buffer: 0xffff6c89e000 (phys 0xa3000000) + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: c7x: Result buffer: 0xffff7549e000 (phys 0xabc00000) + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: [08:05:48] {TVM_HOME}/src/runtime/contrib/tidl/tidl_runtime_rpmsg.cc:216: TVM AM62D: ping status=0 version=0x20000 uptime=17438ms + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: [08:05:48] {TVM_HOME}/src/runtime/contrib/tidl/tidl_runtime_rpmsg.cc:239: TVM AM62D: loading deploy_mod (51527424 bytes) with 1 input name(s) + Oct 23 08:05:49 am62dxx-evm tvm_model_daemon[997]: c7x: Loaded module handle=1 (text=0 data=0 input_offset=51527443) + Oct 23 08:05:49 am62dxx-evm tvm_model_daemon[997]: [08:05:49] {TVM_HOME}/src/runtime/contrib/tidl/tidl_runtime_rpmsg.cc:263: TVM AM62D: module loaded, handle=1 + +To preload artifacts at boot: + +.. code-block:: console + + root@am62dxx-evm:~# systemctl enable --now tvm-model-preload.service + +If the daemon is not running, both the Linux application and GStreamer ``titvm`` element +will wait up to 30 seconds before falling back to loading the model in-process. + +****************** +DSP Firmware Check +****************** + +Both the Linux application and GStreamer pipelines require the C7x™ DSP firmware to be +running. Verify before use: + +.. code-block:: console + + root@am62dxx-evm:~# cat /sys/class/remoteproc/remoteproc0/state + running diff --git a/source/devices/AM62DX/edgeai/out_of_box_demo.rst b/source/devices/AM62DX/edgeai/out_of_box_demo.rst new file mode 100644 index 000000000..7ba25367e --- /dev/null +++ b/source/devices/AM62DX/edgeai/out_of_box_demo.rst @@ -0,0 +1,282 @@ +.. _am62dx_edgeai_oob_demo: + +======================= +Out-of-Box Web GUI Demo +======================= + +The AM62DX EVM ships with an out-of-box web-based GUI that demonstrates EdgeAI +capabilities directly from a browser. Once the board is powered on and connected to +a network, the web portal can be accessed from any device on the same network without +any additional setup. + +The portal showcases the three core EdgeAI pipelines — home screen, speech +enhancement, and audio classification — running on the C7x™ DSP. + +OOB Web GUI Home Screen +======================= + +.. figure:: /images/am62dx_web_gui_home.png + :align: center + :width: 80% + + AM62DX out-of-box web GUI home screen + +This is the Sitara AM62D DSP + Edge AI Portal, a web-based user interface for managing and +evaluating Texas Instruments' AM62D Evaluation Module. The dashboard allows developers +to launch real-time audio analytics and DSP compute demos, and monitor live system +telemetry like CPU load, memory footprints, and SDK software versions. + +To open the EdgeAI portal, follow these steps: + +1. Once the AM62D EdgeAI image is flashed (see :ref:`pub_edgeai_am62dxx_flashing`), get the IP address of the EVM using ``ip addr`` +2. Open a browser and navigate to ``:3000`` +3. This starts the web server. + +The left navigation sidebar starts with ``Home``, then groups the rest into three sections: + +- **Demos** — ``Audio Intelligence`` (speech enhancement and audio classification) and + ``DSP Acceleration`` (2D FFT, biquad filter chain, and workload offload via RPMsg-DMA + compute demos). +- **Tools** — ``AI Model Inspector`` to upload and browse TIDL (TI Deep Learning) model + inspection reports, including architecture, layer details, and performance metrics. +- **System** — ``Device Info``, ``Logs``, and ``Help``. + +The main panel opens with a welcome banner — ``SITARA AM62D DSP + Edge AI Portal`` — and a +**GET STARTED** button that jumps straight to the Audio Intelligence demo. Below it, an +**Explore Demos And Tools** panel opens or shows the same three entry points (Audio +Intelligence, DSP Acceleration, AI Model Inspector) as clickable cards with a one-line +description each. + +The home screen surfaces the board's connection state and identity at a glance: + +- **Connection status** — top-right banner shows ``Connected to AM62D EVM`` along with the + board's IP address. +- **Info panel** — current ``SDK Version``, ``MCU+ SDK Version``, and ``TIDL Version`` + installed on the board. +- **Device Info panel** — ``Device``, ``Board``, ``SoC``, ``IP Address``, and system ``Uptime``. +- **Runtime Status panel** — live ``CPU Load``, ``CPU Avg``, ``CPU Peak``, and ``RAM + Used/Free``, refreshed continuously while the portal is open. + +A status bar at the bottom of every page repeats ``Uptime``, ``CPU Load``, and ``RAM`` for +quick reference while you are deep in a demo page, along with the portal version, build date, +and a link to the full documentation. + +A theme toggle in the top navigation bar switches the entire portal between dark mode +(default) and light mode. The selection applies immediately across all pages and persists +as you navigate between demos. + +Speech Enhancement Demo +======================= + +.. figure:: /images/am62dx_web_gui_speech_enhancement.png + :align: center + :width: 80% + + AM62DX web GUI — speech enhancement demo + +This demonstration showcases real-time speech enhancement. It integrates noise reduction +powered by the C7x™ DSP to significantly improve speech quality. + +Under ``Demo Selection`` on the left, three options are available: + +- **Speech Enhancement** — C7x™ DSP signal processing + GCRN model inference +- **Audio Classification** — C7x™ DSP + YAMNet/VGGish classification models +- **Custom Audio Pipeline** — a generic ``gst-launch`` runner with artifact management for + running custom GStreamer pipelines from the portal. + +The ``About`` and ``Model Info`` collapsible panels describe the demo and the underlying +GCRN-based noise-suppression model. The ``Audio Source`` card shows the active input file +(for example ``input_audio.wav`` — 1ch, 16 kHz, 16-bit, 9.77s) with a ``Default`` tag, and a +status indicator (``Ready``, ``Running``, etc.). + +To run the speech enhancement demo: + +1. Click **Run Demo** in the top-right corner to begin the process. The status line below the + Audio Source card changes from ``Ready`` to running, and the button itself becomes + **Stop Demo**. +2. Allow the system to complete the processing cycle. +3. Once finished, the interface will display both the original input and the processed output audio files. +4. Use the play button on the input file to hear the original noisy audio. +5. Use the play button on the output file to hear the enhanced version. + +**Speech Enhancement Result** + +The ``Visualization`` section plots both a spectrogram and a waveform for the ``Input +(Noisy)`` and ``Output (Enhanced)`` audio, stacked one above the other, with hover tooltips +reporting time, frequency, and spectral level (or sample amplitude for the waveform). +Spectrogram controls let you switch between ``per-plot peak`` (auto-scale each spectrogram independently) +and ``fixed -80 to 0 dB`` color scale, zoom the time axis, and **SAVE ZIP** to download the plotted data +and waveforms for offline analysis. + +.. figure:: /images/am62dx/edgeai/web_speech_enhancement.gif + :align: center + :width: 80% + + AM62DX web GUI — speech enhancement demo Result 1 (input vs. enhanced spectrograms) + + +.. figure:: /images/am62dx_web_gui_speech_enhancement_result_2.png + :align: center + :width: 80% + + AM62DX web GUI — speech enhancement demo Result 2 (input vs. enhanced waveforms, and + per-stage processing times) + +Below the waveforms, a ``Processing Times`` table breaks down DSP/TIDL latency per +processed frame range (for example ``0-400``, ``301-701``, ``602-1002`` — overlapping +ranges reflecting the overlap-save chunking described in +:ref:`pub_edgeai_am62dx_enhancement_chunking`) into ``Pre-Processing (STFT)``, ``Model +Inference (GCRN)``, ``Post-Processing (ISTFT)``, and ``Total Time``. In a typical run, STFT +and ISTFT each take on the order of 10-20 ms per chunk while the GCRN model inference step +dominates the total, so this table is the quickest way to see where latency budget on the +DSP is actually going. + + +Audio Classification Demo +========================= + +.. figure:: /images/am62dx_web_gui_audio_classification.png + :align: center + :width: 80% + + AM62DX web GUI — audio classification demo + +This demo demonstrates audio classification using YAMNet and VGGish models on the AM62D C7x™ DSP. + +The ``Model`` panel lets you pick the ``Classification Model`` — ``YAMNet`` (521 AudioSet +classes, MobileNet v1) or ``VGGish`` (10 UrbanSound8K classes, VGGish features); the panel +shows this class-count and backbone caption under whichever model is selected, so you can +confirm the right one before running. Unlike Speech Enhancement, which defaults to a sample +WAV file, Audio Classification defaults to live input: the ``Audio Source`` card shows +``AM62D2-EVM: pcmdevice-codec-0`` tagged ``Live Microphone`` (live ALSA capture, 16 kHz mono +PCM, 1s chunks). The ``Input Source`` dropdown lets you switch to a different captured device +with ``REFRESH``, or classify a recorded clip instead with ``UPLOAD WAV FILE``. + +To run the audio classification demo: + +1. Select Audio Classification from the Demo Selection menu on the left. +2. Select Model from the model list — YAMNet or VGGish. +3. Select the audio source: leave it on the live microphone, or pick/upload a WAV file. +4. Click **Run Demo** in the top-right corner; the status line shows ``Idle`` before you start. +5. Check the audio classification result in the output section below. + +While running, a live status line (``Running Audio Classification``) replaces ``Idle`` above +the results, and the top-right button becomes **Stop Demo**. The ``Classification Results`` +panel lists each detected event with a running count and a wall-clock timestamp, newest first +(for example ``13 Silence 02:11:06 pm``), and can be cleared with the trash icon. Below it, a +``Top-10 Predictions Over Patches`` heatmap plots the model's top classes against successive +audio patches over time, with a logit-value color scale for confidence — useful for seeing +which classes the model was close to choosing, not just the winning one. + +**YAMNET Model Result** + +.. figure:: /images/am62dx/edgeai/web_audio_classification_yamnet.gif + :align: center + :width: 80% + + AM62DX web GUI — audio classification YAMNet demo. + +**VGGish Model Result** + +.. figure:: /images/am62dx/edgeai/web_audio_classification_vggish.gif + :align: center + :width: 80% + + AM62DX web GUI — audio classification VGGish demo + +Custom Audio GStreamer Pipeline +================================ + +The ``Custom Audio Pipeline`` demo in the sidebar lets you run arbitrary GStreamer +pipelines directly on the AM62D. This is useful for prototyping custom audio processing +chains, debugging GCRN-based models with custom parameters, or testing YAMNet/VGGish +classification on different audio sources. + +To use your own trained model, see :ref:`Integrating a New Model `. + +The interface has three main areas: + +- **Pipeline Presets** — quick-start tabs that load pre-built pipelines (``Speech + Enhancement``, ``Classification (YAMNet)``, ``Classification (VGGish)``, ``Custom``). + Pre-configured pipelines are hidden behind the phrase "No saved pipelines yet — write a + command and click Save As". +- **Pipeline Command** — a full ``gst-launch-1.0`` command editor. Copy and paste buttons + let you transfer the command to the clipboard or load a saved preset. +- **Input Files** and **TVM Artifacts** — collapsible panels for staging input audio files + and model files (TIDL artifacts). + +Below the ``Pipeline Command``, a collapsible ``Input Source`` dropdown shows the device +or file to be piped into the pipeline, and a status indicator shows whether the system is +ready, running, or idle. + +Speech Enhancement Pipeline with GCRN +------------------------------------- + +This preset runs a GCRN-based noise-suppression pipeline using the C7x™ DSP and TIDL runtime. + +**To run the GCRN speech enhancement pipeline:** + +1. Click the ``Speech Enhancement`` preset tab to load the pre-built command. +2. The ``Pipeline Command`` box displays the full GStreamer command. +3. Click ``Run Demo`` to start the pipeline. +4. The ``Pipeline Output`` panel displays processing logs, including DSP execution times, + model inference latency, and phase transitions (``PREROLLED``, ``PLAYING``, etc.). +5. After execution, the output file is ready for playback — download it or use the play + button to verify the noise reduction. + +.. figure:: /images/am62dx/edgeai/web_gst_speech_enhancement_gcrn.gif + :align: center + :width: 80% + + Output from the GCRN pipeline + +Audio Classification with YAMNet +-------------------------------- + +This preset runs YAMNet (AudioSet, 521 classes) using the C7x™ DSP for real-time audio +event classification. + +**To run the YAMNet classification pipeline:** + +1. Click the ``Classification (YAMNet)`` tab. +2. The ``Pipeline Command`` box displays the full GStreamer command for YAMNet classification. +3. Choose the audio source — a file or live microphone — from the ``Input Source`` dropdown. +4. Click ``Run Demo``. +5. The ``Classification Results`` output shows top-3 predictions per audio window with + confidence logits; execution ends after processing the full input. + +.. figure:: /images/am62dx_web_gui_speech_classification_gst_yamnet.png + :align: center + :width: 80% + + YAMNet pipeline command + +.. figure:: /images/am62dx/edgeai/web_gst_audio_classification_yamnet.gif + :align: center + :width: 80% + + YAMNet inference output showing top-3 predictions + +Audio Classification with VGGish +-------------------------------- + +This preset runs VGGish (UrbanSound8K, 10 classes) for environmental sound classification. + +**To run the VGGish classification pipeline:** + +1. Click the ``Classification (VGGish)`` tab. +2. The command is similar to YAMNet but uses VGGish model weights and UrbanSound8K class labels. +3. Select your audio source and click ``Run Demo``. +4. The output lists detected events with confidence scores for each of the 10 UrbanSound8K classes. + +.. figure:: /images/am62dx_web_gui_speech_classification_gst_vggish.png + :align: center + :width: 80% + + VGGish classification pipeline command + +.. figure:: /images/am62dx/edgeai/web_gst_audio_classification_vggish.gif + :align: center + :width: 80% + + VGGish inference output diff --git a/source/devices/AM62DX/edgeai/overview.rst b/source/devices/AM62DX/edgeai/overview.rst new file mode 100644 index 000000000..90cd4bda3 --- /dev/null +++ b/source/devices/AM62DX/edgeai/overview.rst @@ -0,0 +1,54 @@ +.. _am62dx_edgeai_overview: + +######## +Overview +######## + +**Welcome to Processor SDK Linux Edge AI for AM62DX!** + +The SDK provides software and tools to run high-performance deep learning inference +on Texas Instruments' AM62DX processor. It enables a practical Edge AI solution by +combining a high-performance A53 ARM processor with a dedicated DSP optimized for +neural network inference and digital signal processing. This Edge AI inference chain +is a common pattern across TI devices, allowing efficient on-device AI workloads with +real-time performance. The AM62DX specifically supports audio modality to run +audio analytics pipelines such as speech enhancement and audio classification +on the DSP. + +.. figure:: /images/am62dx/edgeai/web_home_screen.gif + :scale: 30 + :align: center + + AM62DX out-of-box web GUI home screen + +.. figure:: /images/edgeai/edgeai-sdk-feature.png + :scale: 50 + :align: center + + Processor SDK Linux Edge AI for AM62DX feature overview + +The SDK enables an interplay of open-source components such as GStreamer and +Apache TVM. The SDK supports editing, building, and debugging applications directly +on the target, as well as on a host PC where applications are cross-compiled and +built before being deployed to the target. + +.. figure:: /images/am62dx_open_src_comp.png + :scale: 50 + :align: center + + Industry Standard Components supported in Processor SDK Linux Edge AI for AM62DX + +The SDK mainly comprises of three parts as shown in the illustration below. + +- **Edge AI application stack** — Linux-side applications and GStreamer pipelines + that drive audio inference using TVM-RT and open-source components. +- **Foundational Linux components** — AArch64 Linux , secure boot, filesystem, and Linux + drivers that form the base platform. +- **MCU+ SDK** — firmware framework for the C7x™ DSP, handles signal chains (STFT/ISTFT, pre/post-processing) + and model dispatch. Includes build system and flashing utilities. + +.. figure:: /images/am62dx_sdk_overview.png + :scale: 80 + :align: center + +To get started with the setup click the **Next** button. diff --git a/source/devices/AM62DX/edgeai/performance.rst b/source/devices/AM62DX/edgeai/performance.rst new file mode 100644 index 000000000..365f23c3e --- /dev/null +++ b/source/devices/AM62DX/edgeai/performance.rst @@ -0,0 +1,136 @@ +.. include:: + +.. _edgeai-performance: + +##################### +Performance Benchmark +##################### + +This section reports **inference performance** - latency and memory - for the +reference |__PART_FAMILY_NAME__| EdgeAI models. The same methodology (perfsim +estimate + on-target measured) applies to any model you compile through the +workflow. It is **not** an accuracy or audio-quality benchmark: model quality +(classification accuracy, PESQ / STOI / SI-SDR) is measured by the ``evaluate`` +step in :ref:`Model Compile and Evaluate `. + +*********** +Methodology +*********** + +Two independent figures are reported for each model, and they must not be +confused: + +.. list-table:: + :header-rows: 1 + :widths: 22 78 + + * - Figure + - How it is obtained + * - **perfsim estimate** + - Produced by the TIDL performance simulator during ``compile``. The + compiler writes a ``result.yaml`` per model under its work directory; + the ``perfsim`` latency, GMAC count, and estimated DDR transfer are + read directly from it. This is a *static estimate* for the C7\ |trade| + NPU subgraph - it does not include Arm-side pre/post-processing or + runtime overhead. + * - **on-target measured** + - Wall-clock latency timed on the AM62D target while running the + :ref:`Model Zoo ` inference scripts. For the + streaming speech-enhancement models it is also expressed as a + **real-time factor (RTF)** = processing time / audio duration; RTF < 1 + means the model keeps up with real-time audio. + +Only models compiled for the C7\ |trade| NPU (TVM-RT + TIDL) have a perfsim +estimate. GTCRN runs FP32 on the Arm core through the ONNX runtime and has no +TIDL subgraph, so perfsim does not apply to it. + +.. note:: + + The perfsim values below are read from ``result.yaml`` and are labelled + *estimated (perfsim)*. On-target measured latency and RTF are marked **TBD** + until measured on the AM62D EVM. + +***************** +Per-model results +***************** + +.. list-table:: + :header-rows: 1 + :widths: 16 12 12 20 14 26 + + * - Model + - Model ID + - Precision + - perfsim latency *(estimated)* + - GMACs + - DDR transfer *(estimated)* + * - VGGish11 + - ``ac-0101`` + - 8-bit + - 5.235 ms + - 1.1515 + - 10.73 MB + * - YAMNet + - ``ac-0201`` + - 8-bit + - 1.990 ms + - 0.0692 + - 3.55 MB + * - GCRN + - ``ase-0201`` + - 16-bit + - TBD + - TBD + - TBD + * - GTCRN + - ``ase-0100`` + - FP32 (Arm) + - N/A + - N/A + - N/A + +VGGish11 and YAMNet are compiled and have ``result.yaml`` perfsim values (shown +above). GCRN is compiled at 16-bit but its ``result.yaml`` is not yet captured, +so its perfsim figures are **TBD**. GTCRN is Arm-only (no TIDL subgraph), so +perfsim is **N/A**. + +On-target measured latency and RTF for all four models are **TBD** - to be +filled in from the Model Zoo inference scripts on the AM62D EVM: + +.. list-table:: + :header-rows: 1 + :widths: 20 20 30 30 + + * - Model + - On target + - Measured latency + - RTF + * - VGGish11 + - C7\ |trade| NPU + - TBD + - - (clip classifier) + * - YAMNet + - C7\ |trade| NPU + - TBD + - - (clip classifier) + * - GCRN + - C7\ |trade| NPU + - TBD + - TBD + * - GTCRN + - Arm core + - TBD + - TBD + +RTF is reported only for the streaming speech-enhancement models (GCRN, GTCRN); +VGGish11 and YAMNet classify a whole clip, so a real-time factor is not +meaningful for them. + +******************* +Scope & Limitations +******************* + +This section does **not** assess model quality. Classification accuracy and +speech-enhancement quality metrics are produced by ``tidlrunner-cli evaluate`` +and documented in +:ref:`Model Compile and Evaluate `. diff --git a/source/devices/AM62DX/edgeai/version.txt b/source/devices/AM62DX/edgeai/version.txt new file mode 100644 index 000000000..9df24b162 --- /dev/null +++ b/source/devices/AM62DX/edgeai/version.txt @@ -0,0 +1 @@ +12_01_00_00 diff --git a/source/devices/AM62DX/index.rst b/source/devices/AM62DX/index.rst index 7072e83c0..83e910da2 100644 --- a/source/devices/AM62DX/index.rst +++ b/source/devices/AM62DX/index.rst @@ -16,6 +16,7 @@ Linux SDK Software Developer's Guide /devices/AM62DX/linux/Overview /devices/AM62DX/linux/Release_Specific /linux/Foundational_Components + /linux/index_Edge_AI /linux/How_to_Guides /devices/AM62DX/index_RTOS /linux/Demo_User_Guides/index_Demos diff --git a/source/images/am62dx/edgeai/pipeline_gcrn.png b/source/images/am62dx/edgeai/pipeline_gcrn.png new file mode 100644 index 000000000..7e1132888 Binary files /dev/null and b/source/images/am62dx/edgeai/pipeline_gcrn.png differ diff --git a/source/images/am62dx/edgeai/pipeline_vggish.png b/source/images/am62dx/edgeai/pipeline_vggish.png new file mode 100644 index 000000000..7d9fb9cf4 Binary files /dev/null and b/source/images/am62dx/edgeai/pipeline_vggish.png differ diff --git a/source/images/am62dx/edgeai/pipeline_yamnet.png b/source/images/am62dx/edgeai/pipeline_yamnet.png new file mode 100644 index 000000000..501ef01fd Binary files /dev/null and b/source/images/am62dx/edgeai/pipeline_yamnet.png differ diff --git a/source/images/am62dx/edgeai/web_audio_classification_vggish.gif b/source/images/am62dx/edgeai/web_audio_classification_vggish.gif new file mode 100644 index 000000000..1bca8758c Binary files /dev/null and b/source/images/am62dx/edgeai/web_audio_classification_vggish.gif differ diff --git a/source/images/am62dx/edgeai/web_audio_classification_yamnet.gif b/source/images/am62dx/edgeai/web_audio_classification_yamnet.gif new file mode 100644 index 000000000..db82e6dcb Binary files /dev/null and b/source/images/am62dx/edgeai/web_audio_classification_yamnet.gif differ diff --git a/source/images/am62dx/edgeai/web_gst_audio_classification_vggish.gif b/source/images/am62dx/edgeai/web_gst_audio_classification_vggish.gif new file mode 100644 index 000000000..591d273ca Binary files /dev/null and b/source/images/am62dx/edgeai/web_gst_audio_classification_vggish.gif differ diff --git a/source/images/am62dx/edgeai/web_gst_audio_classification_yamnet.gif b/source/images/am62dx/edgeai/web_gst_audio_classification_yamnet.gif new file mode 100644 index 000000000..f3fdc661f Binary files /dev/null and b/source/images/am62dx/edgeai/web_gst_audio_classification_yamnet.gif differ diff --git a/source/images/am62dx/edgeai/web_gst_speech_enhancement_gcrn.gif b/source/images/am62dx/edgeai/web_gst_speech_enhancement_gcrn.gif new file mode 100644 index 000000000..638ec2a37 Binary files /dev/null and b/source/images/am62dx/edgeai/web_gst_speech_enhancement_gcrn.gif differ diff --git a/source/images/am62dx/edgeai/web_home_screen.gif b/source/images/am62dx/edgeai/web_home_screen.gif new file mode 100644 index 000000000..7d5d8c1d8 Binary files /dev/null and b/source/images/am62dx/edgeai/web_home_screen.gif differ diff --git a/source/images/am62dx/edgeai/web_speech_enhancement.gif b/source/images/am62dx/edgeai/web_speech_enhancement.gif new file mode 100644 index 000000000..121427d6a Binary files /dev/null and b/source/images/am62dx/edgeai/web_speech_enhancement.gif differ diff --git a/source/images/am62dx_board_config.png b/source/images/am62dx_board_config.png new file mode 100644 index 000000000..e9504d1dd Binary files /dev/null and b/source/images/am62dx_board_config.png differ diff --git a/source/images/am62dx_boot_pin.png b/source/images/am62dx_boot_pin.png new file mode 100644 index 000000000..d4a2bad9c Binary files /dev/null and b/source/images/am62dx_boot_pin.png differ diff --git a/source/images/am62dx_edgeai.png b/source/images/am62dx_edgeai.png new file mode 100644 index 000000000..c56cc4f3f Binary files /dev/null and b/source/images/am62dx_edgeai.png differ diff --git a/source/images/am62dx_open_src_comp.png b/source/images/am62dx_open_src_comp.png new file mode 100644 index 000000000..85d296304 Binary files /dev/null and b/source/images/am62dx_open_src_comp.png differ diff --git a/source/images/am62dx_sdk_overview.png b/source/images/am62dx_sdk_overview.png new file mode 100644 index 000000000..826b6cd5f Binary files /dev/null and b/source/images/am62dx_sdk_overview.png differ diff --git a/source/images/am62dx_web_gui_audio_classification.png b/source/images/am62dx_web_gui_audio_classification.png new file mode 100644 index 000000000..239067bef Binary files /dev/null and b/source/images/am62dx_web_gui_audio_classification.png differ diff --git a/source/images/am62dx_web_gui_audio_classification_vggish_result.png b/source/images/am62dx_web_gui_audio_classification_vggish_result.png new file mode 100644 index 000000000..868400a42 Binary files /dev/null and b/source/images/am62dx_web_gui_audio_classification_vggish_result.png differ diff --git a/source/images/am62dx_web_gui_audio_classification_yamnet_result.png b/source/images/am62dx_web_gui_audio_classification_yamnet_result.png new file mode 100644 index 000000000..f04772509 Binary files /dev/null and b/source/images/am62dx_web_gui_audio_classification_yamnet_result.png differ diff --git a/source/images/am62dx_web_gui_home.png b/source/images/am62dx_web_gui_home.png new file mode 100644 index 000000000..e1d66e205 Binary files /dev/null and b/source/images/am62dx_web_gui_home.png differ diff --git a/source/images/am62dx_web_gui_speech_classification_gst_vggish.png b/source/images/am62dx_web_gui_speech_classification_gst_vggish.png new file mode 100644 index 000000000..4fd74f74c Binary files /dev/null and b/source/images/am62dx_web_gui_speech_classification_gst_vggish.png differ diff --git a/source/images/am62dx_web_gui_speech_classification_gst_yamnet.png b/source/images/am62dx_web_gui_speech_classification_gst_yamnet.png new file mode 100644 index 000000000..c6fdf69e4 Binary files /dev/null and b/source/images/am62dx_web_gui_speech_classification_gst_yamnet.png differ diff --git a/source/images/am62dx_web_gui_speech_enhancement.png b/source/images/am62dx_web_gui_speech_enhancement.png new file mode 100644 index 000000000..89b0a774e Binary files /dev/null and b/source/images/am62dx_web_gui_speech_enhancement.png differ diff --git a/source/images/am62dx_web_gui_speech_enhancement_result_1.png b/source/images/am62dx_web_gui_speech_enhancement_result_1.png new file mode 100644 index 000000000..c5a0006b7 Binary files /dev/null and b/source/images/am62dx_web_gui_speech_enhancement_result_1.png differ diff --git a/source/images/am62dx_web_gui_speech_enhancement_result_2.png b/source/images/am62dx_web_gui_speech_enhancement_result_2.png new file mode 100644 index 000000000..ee1d57b9d Binary files /dev/null and b/source/images/am62dx_web_gui_speech_enhancement_result_2.png differ diff --git a/source/images/am62dx_webgui_2dfft.png b/source/images/am62dx_webgui_2dfft.png new file mode 100644 index 000000000..19dae095e Binary files /dev/null and b/source/images/am62dx_webgui_2dfft.png differ diff --git a/source/images/am62dx_webgui_cascade_biquad.png b/source/images/am62dx_webgui_cascade_biquad.png new file mode 100644 index 000000000..1d718df21 Binary files /dev/null and b/source/images/am62dx_webgui_cascade_biquad.png differ diff --git a/source/images/am62dx_webgui_dspoffload.png b/source/images/am62dx_webgui_dspoffload.png new file mode 100644 index 000000000..5a25f5975 Binary files /dev/null and b/source/images/am62dx_webgui_dspoffload.png differ diff --git a/source/images/audioai/tvm_tidl_flow.svg b/source/images/audioai/tvm_tidl_flow.svg new file mode 100644 index 000000000..7d5920868 --- /dev/null +++ b/source/images/audioai/tvm_tidl_flow.svg @@ -0,0 +1,4 @@ + + + +
ONNX audio model
Model import into TVM
Partitioning
TIDL import API
TVM C7™ DSP
code generation
TIDL network compiler
TVM-RT deployable artifact
(deploy_lib.so + graph.json + params)
compile time (x86)
run time (AM62D)
TVM runtime
TIDL-RT
C7™ NPU
TIDL unsupported ops
TIDL supported ops
Text is not SVG - cannot display \ No newline at end of file diff --git a/source/images/edgeai/edgeai-sdk-feature.png b/source/images/edgeai/edgeai-sdk-feature.png index 9a2a45ab7..19b6262c3 100644 Binary files a/source/images/edgeai/edgeai-sdk-feature.png and b/source/images/edgeai/edgeai-sdk-feature.png differ diff --git a/source/linux/Demo_User_Guides/AM62D_2DFFT_User_Guide.rst b/source/linux/Demo_User_Guides/AM62D_2DFFT_User_Guide.rst index b9735b12d..62c1bc62b 100644 --- a/source/linux/Demo_User_Guides/AM62D_2DFFT_User_Guide.rst +++ b/source/linux/Demo_User_Guides/AM62D_2DFFT_User_Guide.rst @@ -60,7 +60,7 @@ Below figure shows how this demo works: processed output data is available in the shared DDR buffer. - Step 7: A53 reads back processed data from DMA buffer - - A53 copies the processed data from the shared buffer for validation. + - A53 copies the processed data from the shared buffer for validation. - Step 8: Validation and Performance Reporting - The application compares the output data against expected results to verify correctness. @@ -79,12 +79,12 @@ Hardware prerequisites - USB to Universal asynchronous receiver-transmitter (UART) cable for console access - PC (Windows or Linux) to flash image onto an SD Card - Host PC Requirements: - + - Operating system: - + - Windows: |__WINDOWS_SUPPORTED_LONG__| - Ubuntu: |__LINUX_UBUNTU_VERSION_LONG__| - + - Memory: Minimum 4GB RAM (8GB or more recommended) - Storage: At least 10GB of free space @@ -105,10 +105,10 @@ EVM setup ********* #. Cable Connections - + - The figure below shows some important cable connections, ports and switches. - Take note of the location of the "BOOTMODE" switch for SD card boot mode. - + .. figure:: /images/AM62D_evm_setup.png :height: 600 :width: 1000 @@ -116,14 +116,14 @@ EVM setup - Refer `Quick Start Guide for AM62D2-EVM `__ for more details. #. Setup UART Terminal - + - First identify the UART port as enumerated on the host machine. - Connect the EVM and UART cable to the UART to USB port as shown in cable connections. - In Windows, you can use the "Device Manager" to see the detected UART ports: - + - Search "Device Manager" in Windows Search Box in the Windows taskbar. - + - If you do not see any USB serial ports listed in "Device Manager" under "Ports (COM & LPT)", then make sure you have installed the UART to USB driver from `FTDI `__. @@ -131,9 +131,9 @@ EVM setup keep other options to default and set 115200 baud rate. #. Setup SD card Boot Mode - + - EVM SD card boot mode setting: - + - BOOTMODE [ 8 : 15 ] (SW3) = 0100 0000 - BOOTMODE [ 0 : 7 ] (SW2) = 1100 0010 @@ -141,11 +141,47 @@ EVM setup Steps to validate 2D FFT DSP offload demo ***************************************** +There are two ways to run and visualize this demo: + +Using Web GUI +============= + +The AM62D EVM ships with a web-based GUI portal that provides an out-of-box way to +run and visualize the 2D FFT demo from any browser on the same network — no host +utility installation required. + +#. Flash and boot the EVM as described in the EVM setup section above. + +#. Get the EVM IP address: + + .. code-block:: console + + root@am62dxx-evm:~# ifconfig + +#. Open a browser on your host PC and navigate to ``http://:3000`` + +#. On the home screen, click **DSP Acceleration** or select it from the left sidebar + under **Demos**. + +#. Select **2D FFT** from the demo options and click the run button to start the demo. + +#. The portal displays the test result (PASSED/FAILED) along with performance metrics: + DSP load, cycle count, and DDR throughput. + +.. figure:: /images/am62dx_webgui_2dfft.png + :align: center + :width: 80% + + AM62D Web GUI — 2D FFT demo + +Using Console +============= + #. Flash an SD card with the :file:`tisdk-default-image-rt-am62dxx-evm.rootfs.wic.xz` image and follow the instructions provided at :ref:`Create SD Card ` guide. -#. Insert the flashed SD card into the `AUDIO-AM62D-EVM `__ - and power on the TI AUDIO-AM62D-EVM. +#. Insert the flashed SD card into the `AM62D EVM `__ + and power on the TI AM62D EVM. #. Make sure the EVM boot mode switches are positioned for SD card boot as described earlier. @@ -158,30 +194,30 @@ Steps to validate 2D FFT DSP offload demo Just enter "root" to log in. #. Run the 2D FFT DSP offload demo application from the console: - + .. code-block:: console - + root@am62dxx-evm:~# rpmsg_2dfft_example #. The application will execute and display the results: - + .. code-block:: console - + RPMsg based 2D FFT Offload Example - + ***************************************** ***************************************** - + C7x 2DFFT Test PASSED C7x Load: 1% C7x Cycle Count: 327000 C7x DDR Throughput: 0.801656 MB/s - + ***************************************** ***************************************** .. note:: - + The test reports "PASSED" if the computed 2D FFT output matches the expected results within tolerance (0.01), otherwise it reports "FAILED". @@ -209,44 +245,44 @@ Building the linux demo binary from sources #. The source code for the 2D FFT DSP offload demo is available as part of the `rpmsg-dma `__. - + .. code-block:: console - + host# git clone https://github.com/TexasInstruments/rpmsg-dma.git -b main #. Download and Install the AM62D Linux SDK from |__SDK_DOWNLOAD_URL__| following the steps mentioned at :ref:`Download and Install the SDK `. #. Prepare the environment for cross compilation. - + .. code-block:: console - + host# source /linux-devkit/environment-setup #. Compile the source: - + .. code-block:: console - + [linux-devkit]:> cd [linux-devkit]:> cmake -S . -B build; cmake --build build - + - This command builds: - + - The example application :file:`rpmsg_2dfft_example` - + - Transfer the generated files to SD card: - + - The example binary :file:`rpmsg_2dfft_example` to :file:`/usr/bin` - The test input data file :file:`2dfft_input_data.bin` to :file:`/usr/share/2dfft_test_data/` - The expected output data file :file:`2dfft_expected_output_data.bin` to :file:`/usr/share/2dfft_test_data/` - The C7 DSP firmware file :file:`fft2d_linux_dsp_offload_example.c75ss0-0.release.strip.out` to :file:`/lib/firmware/` - + - Optional: - + - To build only the library or only the example, use: - + .. code-block:: console - + cmake -S . -B build -DBUILD_LIB=OFF # disables library build cmake -S . -B build -DBUILD_AUDIO_OFFLOAD_EXAMPLE=OFF # disables audio_offload example build cmake -S . -B build -DBUILD_2DFFT_OFFLOAD_EXAMPLE=OFF # disables 2dfft_offload example build diff --git a/source/linux/Demo_User_Guides/AM62D_Cascade_Biquad_Parametric_EQ_User_Guide.rst b/source/linux/Demo_User_Guides/AM62D_Cascade_Biquad_Parametric_EQ_User_Guide.rst index 08ee5278b..a12b7bfc8 100644 --- a/source/linux/Demo_User_Guides/AM62D_Cascade_Biquad_Parametric_EQ_User_Guide.rst +++ b/source/linux/Demo_User_Guides/AM62D_Cascade_Biquad_Parametric_EQ_User_Guide.rst @@ -161,9 +161,46 @@ EVM Setup Steps to run Cascade Biquad Parametric EQ demo ********************************************** +There are two ways to run and visualize this demo: + +Using Web GUI +============= + +The AM62D EVM ships with a web-based GUI portal that provides an out-of-box way to +run and visualize the Cascade Biquad Parametric EQ demo from any browser on the same +network — no host utility installation required. + +#. Flash and boot the EVM as described in the EVM Setup section above. + +#. Get the EVM IP address: + + .. code-block:: console + + root@am62dxx-evm:~# ifconfig + +#. Open a browser on your host PC and navigate to ``http://:3000`` + +#. On the home screen, click **DSP Acceleration** or select it from the left sidebar + under **Demos**. + +#. Select **Cascade Biquad** from the demo options and click the run button to start + the demo. + +#. The portal displays real-time metrics including DSP load, cycle count, throughput, + and EQ filter controls. + +.. figure:: /images/am62dx_webgui_cascade_biquad.png + :align: center + :width: 80% + + AM62D Web GUI — Cascade Biquad Parametric EQ demo + +Using Python Host Utility +========================= + #. Flash an SD card with the :file:`tisdk-default-image-rt-am62dxx-evm.rootfs.wic.xz` image and follow the instructions provided at :ref:`Create SD Card ` guide. -#. Insert the flashed SD card to `AUDIO-AM62D-EVM `__, connect the audio input/output, Ethernet Cable and power on TI AUDIO-AM62D-EVM. +#. Insert the flashed SD card to `AM62D EVM `__, connect the audio input/output, Ethernet Cable and power on TI AM62D EVM. #. Make sure the EVM boot mode switches position set for SD card boot as described earlier diff --git a/source/linux/Demo_User_Guides/AM62D_Dsp_Offload_User_Guide.rst b/source/linux/Demo_User_Guides/AM62D_Dsp_Offload_User_Guide.rst index b48a25ec5..679de9e17 100644 --- a/source/linux/Demo_User_Guides/AM62D_Dsp_Offload_User_Guide.rst +++ b/source/linux/Demo_User_Guides/AM62D_Dsp_Offload_User_Guide.rst @@ -35,15 +35,15 @@ Below figure shows how this demo works: .. figure:: /images/AM62D_dsp_signal_chain_audio_offload.png :height: 80 :width: 1000 - + - CasadeBiquad Low-Pass Filter: This filter is a three-stage, direct form 1 design with a low-pass cut-off frequency of 10KHz. - + - CasadeBiquad High-Pass Filter: This filter is also a three-stage, direct form 1 design with a high-pass cut-off frequency of 2KHz. - Finite Impulse Response(FIR) Low-Pass Filter: This filter is a 64-tap low-pass filter with a cut-off frequency of 8KHz. - + - Real Fast Fourier Transform(FFT) and Inverse Fast Fourier Transform(IFFT) Real: Perform FFT and IFFT of a real signal. - + - Matrix Transpose: Convert the data format between de-interleaved and interleaved formats within the signal chain. - Step 6: Processed data copied back to shared DMA Buffer (DDR) @@ -90,9 +90,9 @@ Hardware Prerequisites - Windows: |__WINDOWS_SUPPORTED_LONG__| - Ubuntu: |__LINUX_UBUNTU_VERSION_LONG__| - + - Memory: Minimum 4GB RAM (8GB or more recommended) - + - Storage: At least 10GB of free space ****************** @@ -137,7 +137,7 @@ EVM Setup - In windows, you can use the "Device Manager" to see the detected UART ports - Search "Device Manager" in Windows Search Box in the Windows taskbar. - + - If you do not see any USB serial ports listed in "Device Manager" under "Ports (COM & LPT)", then make sure you have installed the UART to USB driver from `FTDI `__. - For A53 Linux console select UART boot port (ex: COM34 in below screenshot), keep other options to default and set 115200 baud rate. @@ -147,16 +147,52 @@ EVM Setup - EVM SD card boot mode setting: - BOOTMODE [ 8 : 15 ] (SW3) = 0100 0000 - + - BOOTMODE [ 0 : 7 ] (SW2) = 1100 0010 **************************************** Steps to validate audio DSP offload demo **************************************** +There are two ways to run and visualize this demo: + +Using Web GUI +============= + +The AM62D EVM ships with a web-based GUI portal that provides an out-of-box way to +run and visualize the DSP offload demo from any browser on the same network — no host +utility installation required. + +#. Flash and boot the EVM as described in the EVM Setup section above. + +#. Get the EVM IP address: + + .. code-block:: console + + root@am62dxx-evm:~# ifconfig + +#. Open a browser on your host PC and navigate to ``http://:3000`` + +#. On the home screen, click **DSP Acceleration** or select it from the left sidebar + under **Demos**. + +#. Select **DSP Offload** from the demo options and click the run button to start the demo. + +#. The portal displays real-time metrics including DSP load, filter status, input/output + spectrum and latency. + +.. figure:: /images/am62dx_webgui_dspoffload.png + :align: center + :width: 80% + + AM62D Web GUI — DSP Offload demo + +Using Python Host Utility +========================= + #. Flash an SD card with the :file:`tisdk-default-image-rt-am62dxx-evm.rootfs.wic.xz` image and follow the instructions provided at :ref:`Create SD Card ` guide. -#. Insert the flashed SD card to `AUDIO-AM62D-EVM `__, connect the 3.5mm jack headset/Speaker, Ethernet Cable and power on TI AUDIO-AM62D-EVM. +#. Insert the flashed SD card to `AM62D EVM `__, connect the 3.5mm jack headset/Speaker, Ethernet Cable and power on TI AM62D EVM. #. Make sure the EVM boot mode switches position set for SD card boot as described earlier @@ -183,14 +219,14 @@ Steps to validate audio DSP offload demo root@am62dxx-evm:~# rpmsg_audio_offload_example #. On host machine launch the :file:`audiomon.py` utility either in IP mode or UART mode - + - IP Mode - + .. code-block:: console # python audmon.py ip Ex: # python audmon.py ip 192.168.0.101 - + - UART mode .. code-block:: console @@ -203,11 +239,11 @@ Steps to validate audio DSP offload demo - Real-time visualization of: - Frame-level average amplitude (dBFS) - + - latency tracking - + - Avg load - + - Input/output FFT spectrum (only in IP mode) - Command interface to toggle features (e.g., enabling/disabling FFT filter) @@ -219,9 +255,9 @@ Steps to validate audio DSP offload demo - For more information refer: `README `_. .. note:: - + Input/output audio spectrum plotting is only supported in IP mode. UART mode supports only metrics and command interface, not audio data streams. - + Below is sample snapshot: .. figure:: /images/AM62D_host_utility_snapshot.png diff --git a/source/linux/Demo_User_Guides/AM62D_EdgeAI_User_Guide.rst b/source/linux/Demo_User_Guides/AM62D_EdgeAI_User_Guide.rst new file mode 100644 index 000000000..c0864f042 --- /dev/null +++ b/source/linux/Demo_User_Guides/AM62D_EdgeAI_User_Guide.rst @@ -0,0 +1,337 @@ +.. _AM62D-edgeai-from-linux-user-guide: + +################## +AM62D Edge AI Demo +################## + +******** +Overview +******** + +This guide describes how to set up, build, and run the Edge AI inference demo on the +Texas Instruments AM62D audio evaluation module (EVM). This demo showcases real-time +audio deep learning inference on the C7x DSP, including speech enhancement and audio +classification pipelines using TVM-compiled models. The following pre-compiled models +are included out of the box: GCRN (speech enhancement), YAMNet (521-class audio event +classification), and VGGish (10-class audio classification). +Below figure shows how this demo works: + +.. figure:: /images/am62dx_edgeai.png + :height: 400 + :width: 1200 + +- Step 1: Copy sample audio data to shared DMA Buffer (DDR) + - The Linux application copies input audio data to a shared DMA buffer located + in DDR memory for communication with the C7x DSP. + +- Step 2: Notify DSP using RPMsg (IPC) + - A53 sends a control message via RPMsg (Remote Processor Messaging) to the C7x + core. This message informs the C7x that new input audio data is available for + processing in the shared DDR buffer. + +- Step 3: C7x receives RPMsg and processes the request + - The C7x DSP receives the RPMsg message, reads the buffer address and parameters, + and prepares for inference processing. + +- Step 4: C7x reads from shared DMA Buffer into L2 SRAM + - The C7x DSP copies the audio data from the DMA buffer (DDR) into its local L2 + SRAM for processing. This operation minimizes access latency compared to reading + directly from DDR. + +- Step 5: Inference execution on DSP + - Below figure shows the inference signal chain executed on C7x: + + - STFT: Converts time-domain audio frames to frequency-domain spectral frames. + - TVM Inference: Runs the selected TVM-compiled model (GCRN, YAMNet, or VGGish) + on the C7x NPU. + - ISTFT: For speech enhancement models, converts enhanced spectral output back to + time-domain audio. + +- Step 6: Inference results copied back to shared DMA Buffer (DDR) + - Once DSP processing is complete, the output (inference results or enhanced audio) + is copied back into the shared DMA buffer. + +- Step 7: C7x notifies A53 using RPMsg + - C7x sends a control message via RPMsg to the A53 core. This message informs the + A53 that processed output data is available in the shared DDR buffer. + +- Step 8: A53 receives RPMsg + - The A53 Linux application receives the RPMsg notification from the C7x confirming + that inference results are ready in the shared DDR buffer. + +- Step 9: A53 reads back results from DMA buffer + - A53 copies the inference results from the shared buffer. Depending on the pipeline: + + - **Speech enhancement**: Enhanced audio is saved to a WAV file or played back + through the TAD5212 DAC via McASP. + - **Audio classification**: Top-k class names and confidence scores are printed to + the console or displayed in the web GUI. + +********************** +Hardware Prerequisites +********************** + +- `AM62D EVM `__ + +- Audio input source (e.g., microphone) connected to J1/J2 (PCM6240 ADC) + +- Audio output device (e.g., headphones/speakers) connected to J3/J4 (TAD5212 DAC) + +- SD card (minimum 16GB) + +- USB Type-C 20W power supply (make sure to use type-C to type-C cable) + +- USB-to-UART cable for console access + +- PC (Windows or Linux) to flash image onto an SD Card + +- Ethernet cable and the expansion board `DP83867-EVM-AM `__ for network access + +- Host PC Requirements: + + - Operating system: + + - Windows: |__WINDOWS_SUPPORTED_LONG__| + + - Ubuntu: |__LINUX_UBUNTU_VERSION_LONG__| + + - Memory: Minimum 4GB RAM (8GB or more recommended) + + - Storage: At least 10GB of free space + +****************** +Software and Tools +****************** + +- TI Processor SDK Linux EdgeAI (AM62Dx) + +- MCU+ SDK for AM62Dx + +- `TI Clang Compiler Toolchain `__ + +- CMake, GCC, make, git, scp, minicom + +- `rpmsg-dma library `__ + +********* +EVM Setup +********* + +#. Cable Connections + + - The figure below shows some important cable connections, ports and switches. + + - Take note of the location of the "BOOTMODE" switch for SD card boot mode. + + .. figure:: /images/AM62D_evm_setup.png + :height: 600 + :width: 1000 + + - Refer `Quick Start Guide for AM62D EVM `__ for more details. + +#. Setup UART Terminal + + - First identify the UART port as enumerated on the host machine. + + - Make sure that the EVM and UART cable are connected to the UART to USB port as shown in Cable Connections. + + - In Windows, you can use the "Device Manager" to see the detected UART ports. + + - If you do not see any USB serial ports listed under "Ports (COM & LPT)", install the UART to USB driver from `FTDI `__. + + - For A53 Linux console, select UART boot port, keep other options to default and set 115200 baud rate. + +#. Setup SD card Boot Mode + + - EVM SD card boot mode setting: + + - BOOTMODE [ 8 : 15 ] (SW3) = 0100 0000 + + - BOOTMODE [ 0 : 7 ] (SW2) = 1100 0010 + +#. Audio Connections + + - Connect audio input source to J1 and J2 (PCM6240 ADC input). + + - Connect audio output to speakers/headphones via J3 and J4 (TAD5212 DAC output). + +************************* +Steps to run Edge AI demo +************************* + +There are two ways to run and visualize this demo: + +Using Web GUI +============= + +The AM62D EVM ships with a web-based GUI portal that provides an out-of-box way to +run speech enhancement and audio classification demos from any browser on the same +network — no additional software installation required. + +#. Flash an SD card with the :file:`tisdk-edgeai-image-am62dxx-evm.rootfs.wic.xz` + image and follow the instructions provided at :ref:`Create SD Card ` guide. + +#. Insert the flashed SD card into the AM62D EVM, connect audio input/output and + Ethernet cable, then power on the EVM. + +#. Make sure the EVM boot mode switches are set for SD card boot as described above. + +#. The EVM should boot and display the Arago login prompt. Enter ``root`` to log in. + +#. Get the EVM IP address: + + .. code-block:: console + + root@am62dxx-evm:~# ifconfig + +#. Open a browser on your host PC and navigate to ``http://:3000`` + +#. On the home screen, click **Audio Intelligence** or select it from the left sidebar + under **Demos**. + +.. figure:: /images/am62dx_web_gui_home.png + :align: center + :width: 80% + + AM62D Web GUI — Audio Intelligence home screen + +#. Select **Speech Enhancement** (GCRN) and click the run button to start noise + reduction on the sample audio input. + +.. figure:: /images/am62dx_web_gui_speech_enhancement.png + :align: center + :width: 80% + + AM62D Web GUI — Speech Enhancement demo + +.. figure:: /images/am62dx_web_gui_speech_enhancement_result_1.png + :align: center + :width: 80% + + AM62D Web GUI — Speech Enhancement result + +#. Select **Audio Classification** (YAMNet or VGGish) and click the run button to + start audio event classification. + +.. figure:: /images/am62dx_web_gui_audio_classification.png + :align: center + :width: 80% + + AM62D Web GUI — Audio Classification demo + +.. figure:: /images/am62dx_web_gui_audio_classification_yamnet_result.png + :align: center + :width: 80% + + AM62D Web GUI — YAMNet classification result + +.. figure:: /images/am62dx_web_gui_audio_classification_vggish_result.png + :align: center + :width: 80% + + AM62D Web GUI — VGGish classification result + +Using Linux Inference Application +================================== + +The ``rpmsg_inference_example`` C++ application provides a command-line interface to +run the same inference pipelines. For full details on building and running the application, +refer to :ref:`C++ Inference Application `. + +#. Flash an SD card with the :file:`tisdk-edgeai-image-am62dxx-evm.rootfs.wic.xz` + image and boot the EVM as described above. + +#. Run speech enhancement (GCRN): + + .. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example \ + /usr/share/tvm_inference/json/pipeline_speech_enhancement.json + +#. Run audio classification (YAMNet): + + .. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example \ + /usr/share/tvm_inference/json/pipeline_audio_classification_yamnet.json + +#. Run audio classification (VGGish): + + .. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example \ + /usr/share/tvm_inference/json/pipeline_audio_classification_vggish.json + +For live ALSA input (classification pipelines only): + + .. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example \ + /usr/share/tvm_inference/json/pipeline_audio_classification_yamnet.json \ + --device plughw:0,0 + +For raw PCM from stdin: + + .. code-block:: console + + root@am62dxx-evm:~# arecord -t raw -f S16_LE -c 1 -r 16000 | \ + rpmsg_inference_example \ + /usr/share/tvm_inference/json/pipeline_audio_classification_yamnet.json \ + --stream + +************************* +How to build Edge AI demo +************************* + +Building Edge AI image from Yocto +================================= + +- To build the Edge AI image, refer :ref:`Processor SDK - Building the SDK with Yocto ` + +Building the Linux inference application from sources +===================================================== + +#. The source code is available as part of the `rpmsg-dma `__ repository. + + .. code-block:: console + + host# git clone https://github.com/TexasInstruments/rpmsg-dma.git -b main + +#. Download and Install the AM62D Linux SDK from |__SDK_DOWNLOAD_URL__| following + the steps mentioned at :ref:`Download and Install the SDK `. + +#. Prepare the environment for cross compilation. + + .. code-block:: console + + host# source /linux-devkit/environment-setup + +#. Compile the source: + + .. code-block:: console + + [linux-devkit]:> cd + [linux-devkit]:> export NEO_TVM_PATH=/path/to/neo-tvm + [linux-devkit]:> cmake -S . -B build -DBUILD_EDGE_AI_EXAMPLE=ON \ + -DBUILD_AUDIO_OFFLOAD_EXAMPLE=OFF \ + -DBUILD_2DFFT_OFFLOAD_EXAMPLE=OFF \ + -DBUILD_SIGNAL_CHAIN_BIQUAD_EXAMPLE=OFF + [linux-devkit]:> cmake --build build + + - This builds: + + - The inference application :file:`rpmsg_inference_example` + - The model daemon :file:`tvm_model_daemon` + + - Transfer the generated files to the EVM: + + .. code-block:: console + + host# scp build/example/edge-ai/rpmsg_inference_example root@:/usr/bin/ + host# scp build/example/edge-ai/tvm_model_daemon root@:/usr/bin/ + +Building the DSP firmware from sources +====================================== + +- Refer to the `MCU+ SDK Documentation `__ +- Refer to the `Edge AI TVM + DSP Multi-Model Signal Chain `__ diff --git a/source/linux/Demo_User_Guides/index_Demos.rst b/source/linux/Demo_User_Guides/index_Demos.rst index 46f9baa6c..8ea17b266 100644 --- a/source/linux/Demo_User_Guides/index_Demos.rst +++ b/source/linux/Demo_User_Guides/index_Demos.rst @@ -18,10 +18,25 @@ The SDK supports the following Out-Of-Box demo-applications Display_Cluster_User_Guide TI_LVGL_Demo_User_Guide GUI_Frameworks_User_Guide + EV_charging_Demo_User_Guide.rst + +AM62D DSP Offload Demos +======================= + +.. toctree:: + :maxdepth: 1 + AM62D_Dsp_Offload_User_Guide.rst AM62D_2DFFT_User_Guide.rst AM62D_Cascade_Biquad_Parametric_EQ_User_Guide.rst - EV_charging_Demo_User_Guide.rst + +AM62D Edge AI Demos +=================== + +.. toctree:: + :maxdepth: 1 + + AM62D_EdgeAI_User_Guide.rst .. ifconfig:: CONFIG_sdk in ('j7_foundational') diff --git a/source/linux/index_Edge_AI.rst b/source/linux/index_Edge_AI.rst index 21aa473ef..587943d10 100644 --- a/source/linux/index_Edge_AI.rst +++ b/source/linux/index_Edge_AI.rst @@ -2,6 +2,8 @@ Edge AI ####### +.. ifconfig:: CONFIG_part_family not in ('AM62DX_family') + The SDK provides software and tools to let the users effectively balance deep learning performance with system power and cost on Texas Instrument's processors for edge AI applications. We offer a practical embedded inference solution for next-generation vehicles, smart cameras, edge AI boxes, and autonomous machines and robots. In addition to general purpose micro processors, AM62Ax has integrated micro controllers, DSP, and accelerators for neural network, image, vision, and multimedia processing. With a few simple steps one can run high performance computer vision and deep learning demos using a live camera and display. The SDK also enables an interplay of multiple open-source components such as GStreamer, OpenVx, OpenCV and deep learning runtime such as TFLite, ONNX and TVM-RT. The reference applications showcase perception based examples such as image classification, object detection and semantic segmentation in both Python and C++ variants. The SDK supports edit-build-debug cycles directly on the target and also on PC to cross compile and build the applications. @@ -10,6 +12,14 @@ Edge AI For more information on edge AI software stack, refer `Edge AI Documentation `_ +.. ifconfig:: CONFIG_part_family in ('AM62DX_family') + + The SDK provides software and tools to let users run high-performance audio deep learning inference on Texas Instruments' AM62DX processor. We offer a practical embedded EdgeAI solution for smart speakers, industrial monitoring, voice interfaces, and acoustic event detection applications. The AM62DX integrates a high-performance A53 ARM processor with a dedicated C7x™ DSP optimized for neural network inference and digital signal processing. With a few simple steps one can run real-time EdgeAI pipelines such as speech enhancement and audio classification on the C7x™ DSP. + + The SDK enables an interplay of open-source components such as GStreamer and Apache TVM. The reference applications showcase EdgeAI examples such as speech enhancement and audio classification. The SDK supports editing, building, and debugging applications directly on the target, as well as on a host PC where applications are cross-compiled and built before being deployed to the target. + + For more information on edge AI software stack, refer `Edge AI Documentation `_ + .. ifconfig:: CONFIG_part_variant in ('J721E') For more information on edge AI software stack, refer `Edge AI Documentation `_