diff --git a/.github/styles/config/vocabularies/PSDK/accept.txt b/.github/styles/config/vocabularies/PSDK/accept.txt index 9611e6dcb..ec68dd68d 100644 --- a/.github/styles/config/vocabularies/PSDK/accept.txt +++ b/.github/styles/config/vocabularies/PSDK/accept.txt @@ -33,9 +33,13 @@ Zink [Mm]ulticast [Rr]enderers? [Ss]carthgap +[Ss]ubgraphs? [Tt]oolchain balenaEtcher bdebstrap fdisk +logits? mmdebstrap +perfsim umount +venv diff --git a/.rstcheck.cfg b/.rstcheck.cfg index 3c3e4dfd8..47e5e029e 100644 --- a/.rstcheck.cfg +++ b/.rstcheck.cfg @@ -3,5 +3,5 @@ ignore_directives = ifconfig,graphviz,grid,grid-item,grid-item-card,card,dropdow # ignore the following languages because they require external compilers and # snippets in this repo depend on external code ignore_languages = c,cpp,c++ -ignore_substitutions = __PART_FAMILY_NAME__,__PART_FAMILY_DEVICE_NAMES__,__PRODUCT_LINE_NAME__,__SDK_BUILD_MACHINE__,__SDK_FULL_NAME__,__SDK_SHORT_NAME__,__SDK_INSTALL_FILE__,__SDK_INSTALL_DIR__,__SDK_DOWNLOAD_URL__,__LINUX_UBUNTU_VERSION_LONG__,__LINUX_UBUNTU_VERSION_SHORT__,__OPTEE_PLATFORM_FLAVOR__,__RTOS_UBUNTU_VERSION_LONG__,__WINDOWS_SUPPORTED_LONG__,__FEATURINGMATRIX__,__SYSFW_CORE_NAME__,__PCIE_BASE_ADDRESS__,__PCIE_DEVICE_ID__,__IMAGE_TYPE__,__SDK_OS_NAME__ +ignore_substitutions = __PART_FAMILY_NAME__,__PART_FAMILY_DEVICE_NAMES__,__PRODUCT_LINE_NAME__,__SDK_BUILD_MACHINE__,__SDK_FULL_NAME__,__SDK_SHORT_NAME__,__SDK_INSTALL_FILE__,__SDK_INSTALL_DIR__,__SDK_DOWNLOAD_URL__,__LINUX_UBUNTU_VERSION_LONG__,__LINUX_UBUNTU_VERSION_SHORT__,__OPTEE_PLATFORM_FLAVOR__,__RTOS_UBUNTU_VERSION_LONG__,__WINDOWS_SUPPORTED_LONG__,__FEATURINGMATRIX__,__SYSFW_CORE_NAME__,__PCIE_BASE_ADDRESS__,__PCIE_DEVICE_ID__,__IMAGE_TYPE__,__SDK_OS_NAME__,__EDGEAI_SDK_VERSION__,trade ignore_messages = (Hyperlink target.*is not referenced\.) diff --git a/conf.py b/conf.py index c84b2b8c9..dc5b183b6 100644 --- a/conf.py +++ b/conf.py @@ -261,6 +261,12 @@ FAMILY = os.environ.get("DEVFAMILY", "") OS = os.environ.get("OS", "") + +# EdgeAI docs prefix every shell command with a "$ " prompt symbol; strip it +# from the copy-button output so copied text is directly runnable. +if OS == "edgeai" and FAMILY == "AM62DX": + copybutton_prompt_text = "$ " + try: globals().update(importlib.import_module(f"configs.{FAMILY}.{FAMILY}_{OS}_tags").__dict__) except ModuleNotFoundError as exc: diff --git a/configs/AM62DX/AM62DX_edgeai_config.txt b/configs/AM62DX/AM62DX_edgeai_config.txt new file mode 100644 index 000000000..66c278a67 --- /dev/null +++ b/configs/AM62DX/AM62DX_edgeai_config.txt @@ -0,0 +1,30 @@ +# General family replacement variables and configuration values for AM62D EdgeAI build + +# This file is processed by Python scripts to define both replacement +# variable and configuration values for a device family build. +# For replacement variables, all variables should be listed on a single +# line, and all the variables should be listed on consecutive lines +# (i.e. no additional blank lines between the replacement variables). +# Similarly, all the configuration values should be listed on individual, +# consecutive lines (with no blank lines between the configuration values). +# +# Note that neither replacement variables nor configuration values may contain +# a colon (i.e. ":") in their name. However, values may contain colons. + +Replacement Variables +--------------------- +'__PART_FAMILY_NAME__' : 'AM62Dx' +'__PART_FAMILY_DEVICE_NAMES__' : 'AM62Dx' +'__PRODUCT_LINE_NAME__' : 'Sitara MPU' +'__SDK_FULL_NAME__' : 'Processor SDK Linux EdgeAI AM62Dx' +'__SDK_SHORT_NAME__' : 'PSDK EdgeAI' +'__SDK_DOWNLOAD_URL__' : '`AM62D-SDK-Download-page `__' +'__EDGEAI_SDK_VERSION__' : '12.1' +'__TIDL_VERSION__' : '11_02_18_00' +'__TIDL_TOOLS_VERSION__' : '11.2.x' + +Configuration Values +-------------------- +'CONFIG_part_family' : 'AM62DX_family' +'CONFIG_part_variant' : 'AM62DX' +'CONFIG_sdk' : 'SITARA' diff --git a/configs/AM62DX/AM62DX_edgeai_tags.py b/configs/AM62DX/AM62DX_edgeai_tags.py new file mode 100644 index 000000000..9ea0dc882 --- /dev/null +++ b/configs/AM62DX/AM62DX_edgeai_tags.py @@ -0,0 +1,6 @@ +# Project name and HTML title +project = u'Processor SDK Linux Edge AI for AM62Dx' +html_title = 'Processor SDK Linux Edge AI for AM62Dx Documentation' + +# The master toctree document. +master_doc = 'devices/AM62DX/edgeai/index' diff --git a/configs/AM62DX/AM62DX_edgeai_toc.txt b/configs/AM62DX/AM62DX_edgeai_toc.txt new file mode 100644 index 000000000..8fe5720bc --- /dev/null +++ b/configs/AM62DX/AM62DX_edgeai_toc.txt @@ -0,0 +1,13 @@ +devices/AM62DX/edgeai/index +devices/AM62DX/edgeai/overview +devices/AM62DX/edgeai/evm_hardware_setup +devices/AM62DX/edgeai/out_of_box_demo +devices/AM62DX/edgeai/models_artifacts_and_setup +devices/AM62DX/edgeai/linux_audio_inference_app +devices/AM62DX/edgeai/getting_started_gstreamer +devices/AM62DX/edgeai/edgeai_model_workflow +devices/AM62DX/edgeai/model_zoo +devices/AM62DX/edgeai/model_compile_evaluate +devices/AM62DX/edgeai/model_inspect +devices/AM62DX/edgeai/integrating_new_model +devices/AM62DX/edgeai/performance diff --git a/configs/AM62DX/AM62DX_linux_toc.txt b/configs/AM62DX/AM62DX_linux_toc.txt index 49de07843..39ec71ce8 100644 --- a/configs/AM62DX/AM62DX_linux_toc.txt +++ b/configs/AM62DX/AM62DX_linux_toc.txt @@ -125,5 +125,6 @@ linux/Demo_User_Guides/index_Demos linux/Demo_User_Guides/AM62D_Dsp_Offload_User_Guide linux/Demo_User_Guides/AM62D_2DFFT_User_Guide linux/Demo_User_Guides/AM62D_Cascade_Biquad_Parametric_EQ_User_Guide +linux/Demo_User_Guides/AM62D_EdgeAI_User_Guide linux/Documentation_Tarball diff --git a/source/devices/AM62DX/edgeai/edgeai_model_workflow.rst b/source/devices/AM62DX/edgeai/edgeai_model_workflow.rst new file mode 100644 index 000000000..3f09d6260 --- /dev/null +++ b/source/devices/AM62DX/edgeai/edgeai_model_workflow.rst @@ -0,0 +1,132 @@ +.. include:: + +.. _edgeai-introduction: + +##################### +EdgeAI Model Workflow +##################### + +This guide covers the workflow for audio deep-learning (DL) models on the +|__PART_FAMILY_NAME__| platform, using |__SDK_FULL_NAME__| +|__EDGEAI_SDK_VERSION__|. It targets embedded developers. You can run, +compile, evaluate, and benchmark audio models on the AM62D. Run pre-compiled +artifacts directly on the target. Or use the 64-bit x86 compile-and-evaluate +workflow. + +Four **reference models** serve as worked examples throughout the guide. The +workflow is not limited to them: the same compile-evaluate-inspect flow is a +bring-your-own-model (BYOM) flow that accepts any supported `ONNX +`__ model. The guide will add more reference models over time. + +The remaining sections cover: + +- :ref:`EdgeAI Model Zoo ` - out-of-box demos on the AM62D + target (inference scripts and Jupyter notebooks) using pre-trained models and + pre-compiled artifacts. +- :ref:`Model Compile and Evaluate ` - the + x86 BYOM workflow built on ``edgeai-tidlrunner``: compile a model for the AM62D + and evaluate accuracy on a data set. +- :ref:`Model Inspect ` - analyze the compiled subgraphs + and per-layer behavior with Model Inspector. +- :ref:`Performance benchmark ` - latency and memory + method with per-model tables. + +******************** +Development workflow +******************** + +The end-to-end path is the same for a reference model or your own model: + +#. **Obtain an ONNX model** - a reference graph shipped with the EdgeAI ModelZoo, + or your own exported ONNX. +#. **Compile** it for the C7\ |trade| Neural Processing Unit (NPU) on an x86 host + (:ref:`Model Compile and Evaluate `). +#. **Evaluate** accuracy or audio-quality metrics on a data set. +#. **Inspect** the compiled subgraphs and per-layer analysis (:ref:`Model + Inspect `). +#. **Run** on the AM62D target using the compiled artifact + (:ref:`EdgeAI Model Zoo `). +#. **Benchmark** latency and memory (:ref:`Performance `). + +**************** +Reference models +**************** + +Four reference models cover two task families. Each is a worked example of the +workflow, delivered as an `ONNX `__ graph. Three of the four +run on the C7\ |trade| NPU through the TVM runtime (TVM-RT). GTCRN is the +exception: it runs on Arm only. See :ref:`the framework overview +` below for details. The same flow accepts your own ONNX +model. The guide will add more reference models over time. + +Audio analysis and classification +================================= + +**VGGish11** (model ID ``ac-0101``) + Audio-clip classification. Input is a log-mel patch of shape ``(1, 1, 64, 126)``; + output is ``(1, 10)`` class logits plus a ``(1, 512)`` embedding. Runs at 8-bit, + fully offloaded to the C7\ |trade| NPU via TVM-RT (a single TIDL subgraph). + +**YAMNet** (model ID ``ac-0201``) + Audio-event classification. Input is a log-mel patch of shape ``(1, 1, 96, 64)``; + output is ``(1, 521)`` class scores. Runs at 8-bit, fully offloaded to the + C7\ |trade| NPU via TVM-RT (a single TIDL subgraph). + +Both classification models are evaluated on the UrbanSound8K data set. + +Speech enhancement +================== + +**GCRN** (model ID ``ase-0201``) + Convolutional-recurrent speech enhancement (noise suppression) over a fixed 4-second + window. Input and output are complex spectrograms of shape ``(1, 2, 401, 161)``. + Runs at 16-bit and is offloaded to the C7\ |trade| NPU via TVM-RT + (``tidl_offload: true``). + +**GTCRN** (model ID ``ase-0100``) + Grouped temporal convolutional-recurrent speech enhancement. Input and output + are complex spectrograms of shape ``(1, 257, T, 2)`` with a dynamic time + dimension ``T``. Runs FP32 on the Arm core through the ONNX runtime + (``tidl_offload: false``) - it is the only supported model that is **not** + offloaded to the C7\ |trade| NPU. + +Both speech-enhancement models are evaluated on the VoiceBank-DEMAND-16k +data set. + +.. _edgeai-tvm-tidl: + +******************** +TVM + TIDL framework +******************** + +The AM62D audio models compile and deploy with **TI TVM**. This is TI's fork +of the `Apache TVM `__ deep-learning compiler. The +target accelerator is the **C7**\ |trade| **NPU** - TI's Neural Processing +Unit. It combines the **C7x DSP** with the **Matrix Multiplication +Accelerator (MMA)**. Older documents call this combination *C7x+MMA*. + +TI TVM adds **TIDL** (TI Deep Learning) as an accelerator backend. TIDL +offloads supported layers to the MMA. TVM generates **C7x DSP** code for any +layers that are left. The MMA and the C7x DSP together form the C7\ |trade| +NPU. The result is one deployable artifact. The **entire model runs on the +C7**\ |trade| **NPU**. + +Compilation runs on an x86 host in three stages: + +#. **Model import** - TVM imports the ONNX graph. +#. **Partitioning** - the TIDL import API splits the graph into TIDL-supported + subgraphs and the rest. The TIDL network compiler builds the subgraphs. TVM + generates C7x DSP code for the rest. +#. **Artifact generation** - TVM packages both parts into one TVM-RT + deployable artifact (:file:`deploy_lib.so` + :file:`deploy_graph.json` + + :file:`deploy_param.params`). + +At run time, the AM62D TVM runtime loads this artifact. It then sends the +offloaded subgraphs to TIDL-RT on the C7\ |trade| NPU. + +.. figure:: /images/audioai/tvm_tidl_flow.svg + :width: 80 % + :align: center + + TVM + TIDL compile-and-deploy flow for the AM62D audio models. Green blocks + are handled by TIDL; white blocks are handled by TVM. diff --git a/source/devices/AM62DX/edgeai/evm_hardware_setup.rst b/source/devices/AM62DX/edgeai/evm_hardware_setup.rst new file mode 100644 index 000000000..e00d9a63c --- /dev/null +++ b/source/devices/AM62DX/edgeai/evm_hardware_setup.rst @@ -0,0 +1,229 @@ +.. _am62dx_edgeai_getting_started_platform: + +================================ +Hardware Setup and Configuration +================================ + +This section covers the hardware setup and software configuration required to prepare +the AM62D EVM for running EdgeAI inference applications. + +Hardware setup +============== + +.. _pub_edgeai_am62dx_evm_setup: + +AM62D EVM +--------- + +The AM62D EVM is a development board for audio AI applications built around the AM62DX +processor with a dedicated C7x™ DSP for neural network inference and digital signal +processing. + +To run the demos on the AM62D EVM you will require: + +- `AM62D EVM `__ +- Minimum 16GB SD card +- USB2.0 Type-C ports (5V‐15V input range) +- USB-to-UART cable for serial console access +- Audio input source (e.g., microphone) connected to J1/J2 3.5mm TRS Port +- Audio output device (e.g., headphones) connected to J3/J4 3.5mm TRS Port +- Required: Ethernet cable for remote access via `DP83867-EVM-AM `__ + +Connect the components to the AM62D EVM as shown below: + +.. figure:: /images/am62dx_board_config.png + :height: 600 + :width: 1000 + :align: center + + AM62D EVM layout showing connectors, switches, and boot mode selection + +Refer to the `AM62D Quick Start Guide `__ +for detailed information about AM62D EVM layout and component locations. For more information, visit AM62Dx EVM `User guide `_ + +Boot Mode Configuration +----------------------- + +Set the BOOTMODE switches to SD card boot mode before powering on: + +- BOOTMODE [8:15] (SW3) = 0100 0000 +- BOOTMODE [0:7] (SW2) = 1100 0010 + +.. figure:: /images/am62dx_boot_pin.png + :height: 500 + :width: 500 + :align: center + + AM62D EVM boot pin configuration + +Audio Connectors +---------------- + +- Connect audio input source to J1 and J2 TRS 3.5mm Jacks (PCM6240 ADC input) (Refer Fig: 2.1) +- Connect audio output to speakers/headphones via J3 and J4 TRS 3.5mm Jacks (TAD5212 DAC output) + +Software setup +============== + +Preparing SD Card Image +----------------------- + +Download the latest Processor SDK Linux AM62DX image from: + +|__SDK_DOWNLOAD_URL__| + +Look for the file: :file:`tisdk-edgeai-image-am62dxx-evm.rootfs.wic.xz` + +Download Balena Etcher from https://www.balena.io/etcher/ and flash the image: + +.. figure:: /images/edgeai/balena_etcher.png + :scale: 100 + :align: center + + Balena Etcher SD card flashing tool + +1. Insert SD card into card reader connected to host PC +2. Launch Balena Etcher +3. Click "Select image" and choose the downloaded ``.wic.xz`` file +4. Click "Select target" and choose the SD card device +5. Click "Flash" to start flashing (this may take 10-15 minutes) +6. Wait for completion confirmation +7. Safely eject the SD card + +.. note:: + + Tested with Balena Etcher version 1.7.0. Some issues reported with version 1.7.2. + If flashing fails, try a different version or use the :command:`dd` command on Linux. + +If using a larger than 16GB SD card, expand the root filesystem to use the full capacity: + +.. code-block:: bash + + #find the SD card device entry using lsblk (Eg: /dev/sdc) + #Unmount the BOOT and rootfs partition before using parted tool + umount /dev/sdX1 + umount /dev/sdX2 + + #Use parted tool to resize the rootfs partition + parted -s /dev/sdX resizepart 2 '100%' + e2fsck -f /dev/sdX2 + resize2fs /dev/sdX2 + + #replace /dev/sdX in above commands with SD card device entry + +.. _pub_edgeai_am62dxx_flashing: + +Power ON and Boot +----------------- + +Ensure that the power supply is disconnected before inserting the SD card. +Once the SD card is firmly inserted and the AM62D EVM is powered ON, the board will +boot in 15-30 seconds. + +Monitor the UART console for boot messages: + +.. code-block:: console + + U-Boot SPL 2026.01-ti-g792d3f648d38 (Sep 29 2026 - 22:45:31 +0000) + SYSFW ABI: 4.0 (firmware rev 0x000c '12.1.2--v12.01.02 (Clever Cat)') + Set clock rates for '/a53@0', CPU: 1250MHz at Speed Grade 'T' + SPL initial stack usage: 13568 bytes + Trying to boot from MMC2 + Authentication passed + Authentication passed + Authentication passed + Authentication passed + Authentication passed + Starting ATF on ARM64 core... + + NOTICE: BL31: v2.15.0(release):v2.15.0 + NOTICE: BL31: Built : 16:29:49, May 28 2026 + + U-Boot SPL 2026.01-ti-g792d3f648d38 (Sep 29 2026 - 22:45:31 +0000) + SYSFW ABI: 4.0 (firmware rev 0x000c '12.1.2--v12.01.02 (Clever Cat)') + . + . + . + . + . + . + Type 'root' to login with superuser privileges (no password will be asked). + am62dxx-evm login: root + +At the login prompt, log in as the root user with no password: + +UART Serial Console +------------------- + +The AM62D EVM provides multiple serial ports via USB. 4 serial ports will come up on +your PC — connect to port 1 or equivalent to get the boot logs. + +For **Linux** use minicom: + +.. code-block:: bash + + sudo minicom -D /dev/ttyUSB1 -c on + +.. note:: + + If you are unable to input from keyboard, disable Hardware Flow Control in minicom: + Press Ctrl+A, then O (Setup), then S (Serial port), and verify Flow Control is off. + +For **Windows** use Tera Term: + +https://learn.sparkfun.com/tutorials/terminal-basics/tera-term-windows + +.. note:: + + Baud rate should be configured to 115200 bps. + +Connect Remotely +---------------- + +You can also access the AM62D EVM via SSH using the IP address shown on the console: + +.. code-block:: bash + + root@am62dxx-evm:~# ifconfig + +From your host PC: + +.. code-block:: console + + host# ssh root@ + +If using Ubuntu 22.04, add the following to ``~/.ssh/config``: + +.. code-block:: bash + + Host 192.168.* + HostKeyAlgorithms=+ssh-rsa + +For a full IDE experience, install Visual Studio Code with the "Remote Development" +extension pack from https://code.visualstudio.com/download and connect via Remote-SSH. + +Verifying the System +-------------------- + +After logging in, verify the system is properly booted: + +Check DSP state (should report 'running'): + +.. code-block:: console + + root@am62dxx-evm:~# cat /sys/class/remoteproc/remoteproc0/state + running + +List RPMsg devices: + +.. code-block:: console + + root@am62dxx-evm:~# ls /dev/rpmsg* + /dev/rpmsg0 /dev/rpmsg1 /dev/rpmsg2 /dev/rpmsg3 /dev/rpmsg_ctrl0 /dev/rpmsg_ctrl1 /dev/rpmsg_ctrl2 + +List DMA heap devices: + +.. code-block:: console + + root@am62dxx-evm:~# ls /dev/dma_heap + carveout_dsp_shared-memories carveout_edgeai_shared-memories default_cma_region linux,cma system diff --git a/source/devices/AM62DX/edgeai/getting_started_gstreamer.rst b/source/devices/AM62DX/edgeai/getting_started_gstreamer.rst new file mode 100644 index 000000000..38305c0d4 --- /dev/null +++ b/source/devices/AM62DX/edgeai/getting_started_gstreamer.rst @@ -0,0 +1,399 @@ +.. _pub_edgeai_am62dx_getting_started: + +######################### +GStreamer Audio Pipelines +######################### + +Introduction +============ + +The GStreamer pipeline framework on AM62DX enables real-time audio AI inference through two custom +plugins: ``tidspkernel`` and ``titvm``. Together, they offload signal processing and neural network +inference to the C7x™ DSP while orchestrating the data flow from Linux. With GStreamer, you can: + +- **Build Custom Pipelines** — Compose audio processing chains using standard GStreamer elements +- **DSP Offload** — Offload DSP signal chains (ex: STFT, ISTFT, and deinterleave/interleave) to the C7x™ DSP +- **TVM Inference** — Run TVM-compiled AI models on the C7x™ NPU via the TVM runtime +- **Flexible I/O** — Read from WAV files, live ALSA input, or other GStreamer sources +- **Direct Command-Line Control** — Use :command:`gst-launch-1.0` to quickly test and prototype pipelines + +Supported Pipelines +=================== + +.. list-table:: + :header-rows: 1 + :widths: 30 30 20 20 + + * - Pipeline + - Use Case + - Default Input + - Output + * - Speech enhancement (GCRN) + - Noise reduction with spectral reconstruction + - :file:`input_audio.wav` + - Enhanced WAV file (playback or file) + * - Audio classification (YAMNet) + - 521-class audio event detection + - :file:`urbansound_16k.wav` + - Top-k class names and scores (console) + * - Audio classification (VGGish) + - 10-class environmental sound classification + - :file:`urbansound_16k.wav` + - Top-k class names and scores (console) + +Prerequisites +============= + +All runtime libraries, model artifacts, sample audio files, model daemon, and DSP firmware +verification are pre-packaged in the AM62DX EdgeAI image. See +:ref:`Models, Artifacts and Setup ` for details. + +To build the GStreamer plugins from source, the following host tools are required: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Tool + - Purpose + * - :command:`meson` / :command:`ninja` + - Build system for GStreamer plugins + * - ARM cross-compiler + - Required only for host-side cross-compilation + * - :command:`pkg-config` + - Dependency resolution during build + + +Native Compilation & Installation on AM62D EVM +---------------------------------------------- + +The plugins are pre-installed on the AM62DX EdgeAI image. If you need to rebuild them +natively on the AM62D EVM, follow the steps below: + +.. code-block:: console + + root@am62dxx-evm:~# cd /opt/edgeai-gst-plugins/ + root@am62dxx-evm:/opt/edgeai-gst-plugins# export SOC=am62d + root@am62dxx-evm:/opt/edgeai-gst-plugins# meson setup build --prefix=/usr -Dpkg_config_path=pkgconfig \ + -Dtvm-plugins=enabled -Dstft-plugins=enabled \ + -Ddl-plugins=disabled -Denable-tidl=disabled + root@am62dxx-evm:/opt/edgeai-gst-plugins# ninja -C build + root@am62dxx-evm:/opt/edgeai-gst-plugins# ninja -C build install + root@am62dxx-evm:/opt/edgeai-gst-plugins# ldconfig + + +Verifying the Installation +-------------------------- + +Execute the following command to verify the tidspkernel GStreamer plugin is installed: + +.. code-block:: console + + root@am62dxx-evm:~# gst-inspect-1.0 tidspkernel + Factory Details: + Rank none (0) + Long-name TI DSP Kernel Transform + Klass Transform/DSP + Description Generic DSP kernel offload (STFT/ISTFT/Deinterleave/Interleave) via RPMsg-DMA + Author Pratham Deshmukh + + Plugin Details: + Name ti + Description GStreamer plugin for TI devices + Filename /usr/lib/gstreamer-1.0/libgstti.so + Version 0.7.0 + License Proprietary + Source module GstTIOVX + Binary package GstTIOVX source release + Origin URL http://ti.com + + GObject + +----GInitiallyUnowned + +----GstObject + +----GstElement + +----GstBaseTransform + +----GstDspKernel + +Execute the following command to verify the titvm GStreamer plugin is installed: + +.. code-block:: console + + root@am62dxx-evm:~# gst-inspect-1.0 titvm + Factory Details: + Rank none (0) + Long-name TI TVM Inference + Klass Transform/ML + Description TVM inference on TI processors with C7x™ DSP acceleration + Author Pratham Deshmukh + + Plugin Details: + Name ti + Description GStreamer plugin for TI devices + Filename /usr/lib/gstreamer-1.0/libgstti.so + Version 0.7.0 + License Proprietary + Source module GstTIOVX + Binary package GstTIOVX source release + Origin URL http://ti.com + + GObject + +----GInitiallyUnowned + +----GstObject + +----GstElement + +----GstBaseTransform + +----GstTiTvm + + +Both commands must print the element details. If an element is missing, check that +:file:`/usr/lib/gstreamer-1.0/libgstti.so` exists and that :command:`ldconfig` has run. + +.. _pub_edgeai_am62dx_model_artifacts: + +GCRN based Speech Enhancement +============================= + +Speech enhancement removes background noise from input audio source while keeping the speech +intelligible — useful for voice calls, meeting recordings, hearables, and any product that +captures speech in a noisy environment. This pipeline runs that speech enhancement full +pipeline on the board: it reads a 16 kHz mono WAV file, converts it to the frequency domain on +the DSP, runs the GCRN model on the result, and converts the enhanced spectrum back to audio +which can be played or saved. + +.. _pub_edgeai_am62dx_enhancement_command: + +Speech Enhancement Pipeline using GCRN +-------------------------------------- + +Execute the following command to run the speech enhancement pipeline. The command writes :file:`/tmp/gst_enhanced.wav` and plays the enhanced speech output through ``autoaudiosink``. Connect headphones to J3/J4 as directed in :ref:`pub_edgeai_am62dx_evm_setup` to listen to the enhanced speech. + +.. code-block:: bash + + INPUT=/usr/share/tvm_inference/input/input_audio.wav + MODEL=/usr/share/tvm_inference/artifacts/gcrn + + gst-launch-1.0 -v \ + filesrc location=$INPUT ! wavparse ! \ + audio/x-raw,format=S16LE,rate=16000,channels=1 ! \ + tidspkernel name=stft msg-type=0x1020 model-path=$MODEL \ + hop-size=160 fft-size=320 window-frames=401 batch-size=64 \ + max-stream-samples=160480 ! \ + tidspkernel name=deinterleave msg-type=0x1040 interleave_direction=0 \ + model-path=$MODEL fft-size=320 window-frames=401 ! \ + titvm model-path=$MODEL ! \ + tidspkernel name=interleave msg-type=0x1040 interleave_direction=1 \ + model-path=$MODEL fft-size=320 window-frames=401 ! \ + tidspkernel name=istft msg-type=0x1030 model-path=$MODEL \ + hop-size=160 fft-size=320 window-frames=401 batch-size=64 \ + max-stream-samples=160480 ! \ + audio/x-raw,format=S16LE,rate=16000,channels=1 ! \ + tee name=t \ + t. ! queue ! wavenc ! filesink location=/tmp/gst_enhanced.wav \ + t. ! queue ! audioconvert ! audioresample ! autoaudiosink + +For a headless run, remove the ``tee`` branch and keep only the file output: + +.. code-block:: bash + + ... ! audio/x-raw,format=S16LE,rate=16000,channels=1 ! \ + wavenc ! filesink location=/tmp/gst_enhanced.wav + +.. figure:: /images/am62dx/edgeai/pipeline_gcrn.png + :scale: 75 + :align: center + + GStreamer data-flow for GCRN speech enhancement: input WAV → STFT → GCRN model inference → ISTFT → output WAV + playback + +.. _pub_edgeai_am62dx_enhancement_chunking: + +.. _pub_edgeai_am62dx_audio_classification: + +YAMNet & VGGish based Audio Classification +========================================== + +Audio classification identifies *what* sound is present in input audio source — a siren, glass +breaking, a dog barking, a machine running — rather than transcribing speech. That makes it +useful for acoustic event detection, environmental and industrial sound monitoring, and +safety or anomaly alerts triggered by sound rather than video. This pipeline runs YAMNet or +VGGish on a 16 kHz mono WAV file on the DSP, and prints the top-k predicted classes and their +scores for each analysis window to the console as the pipeline runs. + +YAMNet and VGGish differ mainly in scope: YAMNet recognizes 521 general sound classes, while +VGGish is a smaller, 10-class model. Pick YAMNet for broad coverage, or VGGish when you only +need to distinguish a small, fixed set of sounds and want a lighter model. + +Audio Classification Pipeline using YAMNet +------------------------------------------ + +Execute the following command to run GStreamer based YAMNet audio classification pipeline: + +.. code-block:: bash + + gst-launch-1.0 -v filesrc location=/usr/share/tvm_inference/input/urbansound_16k.wav ! \ + wavparse ! audio/x-raw,format=S16LE,rate=16000,channels=1 ! \ + tidspkernel name=stft msg-type=0x1020 \ + model-path=/usr/share/tvm_inference/artifacts/yamnet \ + hop-size=160 window-frames=96 batch-size=64 ! \ + titvm model-path=/usr/share/tvm_inference/artifacts/yamnet \ + class-map-path=/usr/share/tvm_inference/labels/yamnet_label_list.txt \ + top-k=3 ! \ + fakesink + +YAMNet Command Output +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: text + + root@am62dxx-evm:~# gst-launch-1.0 -v filesrc location=/usr/share/tvm_inference/input/urbansound_16k.wav ! \ + Setting pipeline to PAUSED ... + [TVM] Live top-3 prediction printing enabled (521 classes from /usr/share/tvm_inference/labels/yamnet_label_list.txt) + eptdev = rpmsg-char-8-1455 + Created endpt device rpmsg-char-8-1455, fd = 7 port = 1027 + Pipeline is PREROLLING ... + /GstPipeline:pipeline0/GstWavParse:wavparse0.GstPad:src: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + /GstPipeline:pipeline0/GstCapsFilter:capsfilter0.GstPad:src: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + /GstPipeline:pipeline0/GstDspKernel:stft.GstPad:src: caps = application/octet-stream + /GstPipeline:pipeline0/GstTiTvm:titvm0.GstPad:src: caps = application/octet-stream + /GstPipeline:pipeline0/GstFakeSink:fakesink0.GstPad:sink: caps = application/octet-stream + /GstPipeline:pipeline0/GstTiTvm:titvm0.GstPad:sink: caps = application/octet-stream + /GstPipeline:pipeline0/GstDspKernel:stft.GstPad:sink: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + /GstPipeline:pipeline0/GstCapsFilter:capsfilter0.GstPad:sink: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + Pipeline is PREROLLED ... + Setting pipeline to PLAYING ... + Redistribute latency... + New clock: GstSystemClock + Top-3 predictions for window 1: + 1. Speech: 0.532749 + 2. Music: -4.261989 + 3. Narration, monologue: -4.794737 + Top-3 predictions for window 2: + 1. Music: -1.598246 + 2. Speech: -2.663743 + 3. Television: -3.729240 + Top-3 predictions for window 3: + 1. Music: -1.065497 + 2. Animal: -2.663743 + 3. Speech: -3.196491 + Top-3 predictions for window 4: + 1. Speech: -1.598246 + 2. Music: -1.598246 + 3. Country: -5.860234 + Top-3 predictions for window 5: + 1. Speech: -2.130994 + 2. Music: -3.196491 + 3. Silence: -4.261989 + Top-3 predictions for window 6: + 1. Silence: 14.916960 + 2. Speech: -11.187720 + 3. Inside, small room: -15.982457 + Top-3 predictions for window 7: + 1. Silence: 12.253218 + 2. Speech: -10.122223 + 3. Inside, small room: -15.449709 + Top-3 predictions for window 8: + 1. Silence: 15.449709 + 2. Speech: -10.654972 + 3. Inside, small room: -14.916960 + Got EOS from element "pipeline0". + Execution ended after 0:00:00.302017695 + Setting pipeline to NULL ... + Freeing pipeline ...`` + +.. figure:: /images/am62dx/edgeai/pipeline_yamnet.png + :scale: 75 + :align: center + + GStreamer data-flow for YAMNet based audio classification: input WAV → STFT → YAMNet model inference → console output + +Audio Classification Pipeline using VGGish +------------------------------------------ + +Execute the following command to run GStreamer based VGGish audio classification pipeline: + +.. code-block:: bash + + gst-launch-1.0 -v filesrc location=/usr/share/tvm_inference/input/urbansound_16k.wav ! \ + wavparse ! audio/x-raw,format=S16LE,rate=16000,channels=1 ! \ + tidspkernel name=stft msg-type=0x1020 \ + model-path=/usr/share/tvm_inference/artifacts/vggish \ + hop-size=512 window-frames=126 batch-size=64 ! \ + titvm model-path=/usr/share/tvm_inference/artifacts/vggish \ + class-map-path=/usr/share/tvm_inference/labels/vggish_label_list.txt \ + top-k=3 ! \ + fakesink + +VGGish Command Output +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: text + + Setting pipeline to PAUSED ... + [TVM] Live top-3 prediction printing enabled (10 classes from /usr/share/tvm_inference/labels/vggish_label_list.txt) + eptdev = rpmsg-char-8-1295 + Created endpt device rpmsg-char-8-1295, fd = 7 port = 1027 + Pipeline is PREROLLING ... + /GstPipeline:pipeline0/GstWavParse:wavparse0.GstPad:src: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + /GstPipeline:pipeline0/GstCapsFilter:capsfilter0.GstPad:src: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + /GstPipeline:pipeline0/GstDspKernel:stft.GstPad:src: caps = application/octet-stream + /GstPipeline:pipeline0/GstTiTvm:titvm0.GstPad:src: caps = application/octet-stream + /GstPipeline:pipeline0/GstFakeSink:fakesink0.GstPad:sink: caps = application/octet-stream + /GstPipeline:pipeline0/GstTiTvm:titvm0.GstPad:sink: caps = application/octet-stream + /GstPipeline:pipeline0/GstDspKernel:stft.GstPad:sink: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + /GstPipeline:pipeline0/GstCapsFilter:capsfilter0.GstPad:sink: caps = audio/x-raw, format=(string)S16LE, layout=(string)interleaved, channels=(int)1, rate=(int)16000 + Pipeline is PREROLLED ... + Setting pipeline to PLAYING ... + Redistribute latency... + New clock: GstSystemClock + Top-3 predictions for window 1: + 1. Street music: 8.411190 + 2. Children playing: 8.206039 + 3. Air conditioner: -4.513321 + Got EOS from element "pipeline0". + Execution ended after 0:00:00.082239915 + Setting pipeline to NULL ... + Freeing pipeline ... + + +.. figure:: /images/am62dx/edgeai/pipeline_vggish.png + :scale: 75 + :align: center + + GStreamer data-flow for VGGish based audio classification: input WAV → STFT → VGGish model inference → console output + +Troubleshooting +=============== + +.. list-table:: + :header-rows: 1 + :widths: 40 60 + + * - Issue + - Resolution + * - ``max-stream-samples must be explicitly set (> 0)`` + - Set ``max-stream-samples`` on the STFT element. + * - ``padded length ... exceeds dma_input capacity`` + - Increase ``max-stream-samples`` to the padded length. + * - ``Invalid incoming buffer`` + - The input is longer than ``max-stream-samples``. Increase it. + * - ``Model daemon handshake failed`` followed by a long wait + - Start the daemon, or accept the in-process load time. + * - ``c7x_client_open() failed — is the c7x_compute DSP firmware running?`` + - The DSP firmware is not running. Check ``/sys/class/remoteproc/remoteproc0/state``. + If it shows ``offline``, reboot the evm. After the reboot, + rerun ``/sys/class/remoteproc/remoteproc0/state`` it should now show ``running`` + * - ``DMA heap alloc failed`` + - Not enough contiguous memory for the model. Reboot and retry, then check ``/proc/meminfo``. + * - ``TI-Offload exchange failed`` or ``DSP kernel failed: remote status=`` + - The DSP firmware does not match the plugin. Check the DSP firmware version. + * - ``Failed to set pipeline to PAUSED`` + - Run again with ``GST_DEBUG=tidspkernel:6,titvm:6`` and check the first error. + * - ``Failed to load class-map-path`` + - Check the path, and that the file is readable. + * - Banner prints but no predictions appear + - Run with ``GST_DEBUG=titvm:5``. Check that the pipeline reached PLAYING. + * - Predictions do not match the expected labels + - Check that the model and label file are a pair: YAMNet with :file:`yamnet_label_list.txt`, + VGGish with :file:`vggish_label_list.txt`. + * - ``top-k`` rejected + - Use a value from 1 to 521. + * - Long wait before banner prints + - ``titvm`` is waiting for the model daemon. Start it or accept the in-process load time. diff --git a/source/devices/AM62DX/edgeai/index.rst b/source/devices/AM62DX/edgeai/index.rst new file mode 100644 index 000000000..e77c24329 --- /dev/null +++ b/source/devices/AM62DX/edgeai/index.rst @@ -0,0 +1,30 @@ +.. _edgeai-index: + +###################################################### +Processor SDK Linux Edge AI Software Developer’s Guide +###################################################### + +.. rubric:: Welcome to the AM62D EdgeAI Software Developer's Guide + +.. toctree:: + :maxdepth: 5 + :numbered: + + overview + evm_hardware_setup + out_of_box_demo + models_artifacts_and_setup + linux_audio_inference_app + getting_started_gstreamer + edgeai_model_workflow + model_zoo + model_compile_evaluate + model_inspect + integrating_new_model + performance + +| + ++--------------------------------+--------------------------------------------------------------------------------------------------+ +| .. Image:: /images/E2e.jpg | For technical support please post your questions at `http://e2e.ti.com `__. | ++--------------------------------+--------------------------------------------------------------------------------------------------+ diff --git a/source/devices/AM62DX/edgeai/integrating_new_model.rst b/source/devices/AM62DX/edgeai/integrating_new_model.rst new file mode 100644 index 000000000..60d836f7d --- /dev/null +++ b/source/devices/AM62DX/edgeai/integrating_new_model.rst @@ -0,0 +1,694 @@ +.. _am62dx_edgeai_integrating_new_model: + +####################### +Integrating a New Model +####################### + +This page covers what is needed to integrate a new deep learning model into the AM62DX +Edge AI system. Compiling the model and updating the DSP firmware are required for all +use cases. After that, you can run the model using either the Linux inference application +or GStreamer pipelines — both are independent paths. + +******** +Overview +******** + +- **Compile the Model** — required; generate TVM artifacts for the C7x™ DSP +- **DSP Firmware Changes** — required; register the model in the DSP signal chain +- **Linux Application** — use the ``rpmsg_inference_example`` application +- **GStreamer** — use ``tidspkernel`` and ``titvm`` GStreamer elements + +***************** +Compile the Model +***************** + +Use the TVM compilation workflow to produce three artifacts for the C7x™ DSP: + +- :file:`deploy_graph.json` — graph description with input/output shapes +- :file:`deploy_lib.so` — compiled model library +- :file:`deploy_param.params` — model weights + +See :ref:`Model Compile and Evaluate `. + +******************** +DSP Firmware Changes +******************** + +The DSP firmware selects models at runtime via a ``ModelId`` enum and a dispatcher +``switch``. Adding a new model means adding one entry to the enum and one ``case`` to +the dispatcher. The full step-by-step guide is in the MCU+ SDK Edge AI Add New Model +documentation. + +Files to modify or create +========================= + +.. list-table:: + :header-rows: 1 + :widths: 45 55 + + * - File + - Change + * - ``common/model_config.h`` + - Add ``MODEL_NEWMODEL`` to the ``ModelId`` enum before ``MODEL_COUNT``; bump + ``MODEL_COUNT`` + * - ``common/models//TISP__signal_chain_memory_map.hpp`` + - New file: STFT/log-mel parameter macros (FFT size, hop size, channel count, + buffer sizes) + * - ``common/models//stft_signal_chain_.hpp`` + - New file: declare ``StftSignalChain`` class interface + * - ``common/models//stft_signal_chain_.cpp`` + - New file: implement the signal chain, ``SignalChainOps`` table, and + ``extern "C" newmodel_get_ops()`` + * - ``common/signal_chain_dispatch.cpp`` + - Add ``extern "C"`` declaration and new ``case MODEL_NEWMODEL`` in + ``signal_chain_get_ops()`` + * - ``am62dx-evm/.../example.projectspec`` + - Add ``-I`` include path and ```` entry for the new model source + * - ``am62dx-evm/.../makefile`` + - Add to ``FILES_cpp_common``, ``FILES_PATH_cpp_common``, and ``INCLUDES_common`` + +Key functions to implement in ``stft_signal_chain_.cpp`` +=============================================================== + +- ``create_pre_graph(int16_t *input_addr)`` — allocate arena buffers, generate STFT + analysis window (or log-mel filterbank), build a ``TISP::opVec`` of + ``TypeConversion`` + ``Stft`` (or ``LogMelSpectrum``) +- ``execute_pre_graph(void *pre_graph, int16_t *input_addr)`` — bind input address via + ``setAddr(..., TISP::kAddrModeIn)`` and call ``TISP::execute::graph()`` +- ``delete_pre_graph(void *pre_graph)`` — delete the ``TISP::opVec`` +- ``create_post_graph`` / ``execute_post_graph`` / ``delete_post_graph`` — only for + reconstruction-style models; set to ``NULL`` in ``SignalChainOps`` for analysis-only + models +- ``newmodel_get_preprocess_buf()`` / ``newmodel_get_postprocess_buf()`` — return arena + buffer addresses +- ``newmodel_reset_arenas()`` — call ``signal_chain_arena_reset_ready()`` +- ``newmodel_get_ops()`` — expose the static ``SignalChainOps`` table + +Two model types are supported +============================= + +- **Reconstruction-style** (e.g. DCCRN, GTCRN, GCRN): STFT pre-graph + ISTFT + post-graph; implement all six methods +- **Analysis-only** (e.g. VGGish, YAMNet): STFT/log-mel pre-graph only; post-graph + methods are ``NULL`` in the ``SignalChainOps`` table + +***************** +Linux Application +***************** + +Using a New Model with an Existing Pipeline Type +================================================ + +To bring your own model, first compile it for the AM62DX C7x™ NPU using the +:ref:`Model Compile and Evaluate ` workflow, which produces +the :file:`deploy_graph.json`, :file:`deploy_lib.so` and :file:`deploy_param.params` artifacts. + +Once you have the compiled artifacts: + +#. Deploy the model artifacts to the target: + + .. code-block:: console + + root@am62dxx-evm:~# mkdir -p /usr/share/tvm_inference/artifacts/my_model/ + root@am62dxx-evm:~# cp deploy_graph.json deploy_lib.so deploy_param.params \ + /usr/share/tvm_inference/artifacts/my_model/ + +#. Copy the nearest existing JSON pipeline file as a starting point and adjust it for + your model. The JSON files in ``example/edge-ai/json_files/`` are the authoritative + reference. Key fields to update: + + .. code-block:: json + + { + "pipeline_type": "audio_classification", + "input_file": "/usr/share/tvm_inference/input/my_audio.wav", + "artifacts_path": "/usr/share/tvm_inference/artifacts/my_model", + "labels_path": "/usr/share/tvm_inference/labels/my_model_labels.txt", + "num_classes": 10, + "dsp_config": { + "proc_id": 8, + "endpoint": 13 + }, + "stages": [ + { + "stage_id": "stft_analysis", + "service": "dsp", + "message_type": "C7X_MSG_STFT_ANALYZE", + "parameters": { + "selected_model": 4, + "hop_size": 160, + "model_elems": 64, + "total_frames": 96, + "batch_n": 64 + } + }, + { + "stage_id": "tvm_inference", + "service": "tvm", + "message_type": "TVM_INFERENCE", + "parameters": { + "input_shape": "1,1,96,64" + } + } + ] + } + + Keep ``proc_id``, ``endpoint``, and DSP ``message_type`` values consistent with + the deployed DSP firmware. Adjust ``selected_model``, ``model_elems``, ``total_frames`` + and ``input_shape`` to match your model's spectral layout. + +#. If your model outputs classifications, provide a labels file (one label per line): + + .. code-block:: console + + root@am62dxx-evm:~# cat > /usr/share/tvm_inference/labels/my_model_labels.txt << EOF + Label 0 + Label 1 + ... + EOF + +#. Run the application with the new pipeline: + + .. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example /path/to/my_pipeline.json \ + --input-file /path/to/test_audio.wav + +Adding a New Pipeline Type in C++ +================================= + +If the existing pipeline types (``speech_enhancement``, ``audio_classification``, +``stft_istft``, ``tvm_only``) do not cover your use case, you can add a new pipeline +type. The dispatch is in ``pipeline_manager.cpp`` and follows the same pattern for +every pipeline. + +**Add a header and source file** + +Create ``example/edge-ai/include/my_pipeline.h``: + +.. code-block:: cpp + + #pragma once + #include "pipeline_manager.h" + #include "dsp_task_client.h" + #include "tvm_inference_client.h" + + PipelineManager::CommandResult run_my_pipeline( + PipelineManager::State& state, + DspTaskClient& dsp_client, + TvmInferenceClient& tvm_client, + bool debug); + +Create ``example/edge-ai/src/my_pipeline.cpp``: + +.. code-block:: cpp + + #include "my_pipeline.h" + #include "pipeline_common.h" + #include + + PipelineManager::CommandResult run_my_pipeline( + PipelineManager::State& state, + DspTaskClient& dsp_client, + TvmInferenceClient& tvm_client, + bool debug) + { + // Access pipeline config and stages from state + const auto& config = state.pipeline_config; + const auto& stages = config.stages; + + // Find your custom stage by message_type + const PipelineManager::PipelineStage* my_stage = nullptr; + for (const auto& s : stages) { + if (s.message_type == "MY_CUSTOM_MSG") { + my_stage = &s; + break; + } + } + if (!my_stage) + throw PipelineError{"my_pipeline requires a MY_CUSTOM_MSG stage"}; + + // Read stage parameters + size_t hop_size = std::stoul(my_stage->parameters.at("hop_size")); + + // ... run DSP and TVM stages using dsp_client and tvm_client ... + + std::cout << "[App] my_pipeline complete" << std::endl; + return PipelineManager::CommandResult::SUCCESS; + } + +**Register the pipeline type in** ``pipeline_manager.cpp`` + +In ``run_from_json_file()``, add your new ``pipeline_type`` string to the dispatch +block alongside the existing types: + +.. code-block:: cpp + + #include "my_pipeline.h" // add this include at the top + + // ... inside run_from_json_file(), in the dispatch block: + if (pipeline_type == "speech_enhancement") { + result = run_speech_enhancement_pipeline(state_, *generic_client_, *tvm_client_, debug_); + } else if (pipeline_type == "audio_classification") { + result = run_audio_classification_pipeline(state_, *generic_client_, *tvm_client_, debug_); + } else if (pipeline_type == "tvm_only") { + result = run_tvm_pipeline(state_, *tvm_client_); + } else if (pipeline_type == "stft_istft") { + result = run_stft_istft_pipeline(state_, *generic_client_, debug_); + } else if (pipeline_type == "my_pipeline") { // add this + result = run_my_pipeline(state_, *generic_client_, *tvm_client_, debug_); + } else { + std::cout << "[App] Error: Unknown pipeline_type: " << pipeline_type << std::endl; + return -1; + } + +**Add the source file to** :file:`CMakeLists.txt` + +In :file:`example/edge-ai/CMakeLists.txt`, add :file:`src/my_pipeline.cpp` to the +``EDGE_AI_SOURCES`` list: + +.. code-block:: cmake + + set(EDGE_AI_SOURCES + src/main.cpp + src/tvm_inference_client.cpp + src/dsp_task_client.cpp + src/pipeline_manager.cpp + src/pipeline_common.cpp + src/audio_utils.cpp + src/speech_enhancement_pipeline.cpp + src/audio_classification_pipeline.cpp + src/tvm_pipeline.cpp + src/stft_istft_pipeline.cpp + src/my_pipeline.cpp # add this + ) + +**Create a JSON pipeline file for the new type** + +.. code-block:: json + + { + "pipeline_type": "my_pipeline", + "input_file": "/usr/share/tvm_inference/input/my_audio.wav", + "artifacts_path": "/usr/share/tvm_inference/artifacts/my_model", + "dsp_config": { "proc_id": 8, "endpoint": 13 }, + "stages": [ + { + "stage_id": "my_stage", + "service": "dsp", + "message_type": "MY_CUSTOM_MSG", + "parameters": { "hop_size": "160" } + }, + { + "stage_id": "tvm_inference", + "service": "tvm", + "message_type": "TVM_INFERENCE", + "parameters": { "input_shape": "1,1,96,64" } + } + ] + } + +Rebuild with ``cmake --build build`` and run: + +.. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example /path/to/my_pipeline.json + +Debugging and Troubleshooting +============================= + +**System checks** + +.. code-block:: console + + root@am62dxx-evm:~# grep . /sys/class/remoteproc/remoteproc*/name + root@am62dxx-evm:~# grep . /sys/class/remoteproc/remoteproc*/state + root@am62dxx-evm:~# ls -l /dev/rpmsg* /dev/dma_heap/* + root@am62dxx-evm:~# ldd /usr/bin/rpmsg_inference_example + root@am62dxx-evm:~# find /usr/share/tvm_inference -maxdepth 3 -type f + +**DSP firmware trace** + +.. code-block:: console + + root@am62dxx-evm:~# cat /sys/kernel/debug/remoteproc/remoteproc*/trace0 + +**Enable debug logging** + +.. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example --debug + +.. list-table:: + :header-rows: 1 + :widths: 40 60 + + * - Symptom + - Action + * - ``rpmsg_inference_example`` not found + - Check binary is in ``/usr/bin/`` and has execute permission. + Run :command:`ldd` to check for missing shared libraries. + * - Connection to DSP endpoint fails + - Verify DSP state is ``running``. Check :file:`/dev/rpmsg*` exists. + View DSP trace with :command:`trace0`. + * - Model artifact path not found + - Verify ``artifacts_path`` in the JSON matches the deployed directory. + Check :file:`deploy_graph.json`, :file:`deploy_lib.so` and :file:`deploy_param.params` are present. + * - Audio input device not found + - Run :command:`arecord -l` to list ALSA devices. Verify ``--device`` name matches. + * - Long wait before inference starts + - ``tvm_model_daemon`` is not running. Start it or accept the in-process load time. + +********* +GStreamer +********* + +.. _pub_edgeai_am62dx_byom: + +Bring-Your-Own Model +==================== + +The three reference pipelines use pre-trained, pre-compiled models so you can see the +plugins working immediately. Most production use cases, though, need a model trained on +your own data — your own set of sound events, or a speech-enhancement model tuned to +your product's microphone and acoustic environment. This section covers what it takes to +run such a model through ``tidspkernel`` and ``titvm`` once it has been compiled for the +AM62Dx C7x™ DSP. + +Compiling the model itself — taking an ONNX graph through quantization and generating +:file:`deploy_graph.json`, :file:`deploy_lib.so` and :file:`deploy_param.params` — is covered +separately in :ref:`Model Compile and Evaluate `. This section +assumes you already have those three artifacts and focuses on wiring them into a GStreamer +pipeline. + +Requirements +------------ + +A compiled model directory with the same three files the reference models use: + +.. code-block:: text + + my_model/ + ├── deploy_graph.json graph description; input/output shapes read from attrs.shape + ├── deploy_lib.so compiled model library + └── deploy_param.params model weights + +Beyond these files, you must specify two critical model properties that the plugins cannot +auto-detect: + +- **Task type**: whether the model performs speech enhancement (like GCRN) or + classification (like YAMNet/VGGish). This determines which pipeline you should adapt. +- **Spectral layout**: the FFT size, hop size, and window length the model was trained + with, plus the number of spectral coefficients per frame that the DSP firmware must + produce. + +Example: A New Model +-------------------- + +Suppose you have a new model with a unique architecture and spectral front-end, and +DSP firmware ModelId ``5`` has already been added to support it. To use it in a GStreamer +pipeline, follow the same pattern as the YAMNet/VGGish examples, but explicitly set all +spectral properties instead of relying on path-based auto-detection: + +.. code-block:: bash + + MODEL=/path/to/my_model + gst-launch-1.0 -v filesrc location=$INPUT ! \ + wavparse ! audio/x-raw,format=S16LE,rate=16000,channels=1 ! \ + tidspkernel name=stft msg-type=0x1020 model-path=$MODEL \ + selected-model=5 model-elems=128 hop-size=160 fft-size=256 \ + window-frames=100 batch-size=64 ! \ + titvm model-path=$MODEL \ + class-map-path=/path/to/my_model/labels.txt top-k=3 ! \ + fakesink + +This command uses the same approach as the reference models — the difference is that all +spectral parameters (``selected-model``, ``model-elems``, ``fft-size``, ``hop-size``, +``window-frames``) are now explicitly set for your model rather than auto-detected from +the path. The ``titvm`` element requires no modifications beyond ``model-path`` — it +automatically loads and runs whatever TVM artifacts it finds in that directory. + +For a speech-enhancement model with a new architecture, use the same approach: build +your pipeline with the ``deinterleave`` → ``tidspkernel`` (STFT) → ``titvm`` → +``interleave`` → ``tidspkernel`` (ISTFT) chain from the GCRN reference pipeline, then +set the ``tidspkernel`` properties to match your model's spectral layout instead of +GCRN's. + +Supporting New Model Architectures in the Plugins +================================================= + +If your model introduces a new architecture or spectral processing scheme not covered by +existing DSP firmware ModelIds, the DSP DSP firmware must first add support for that new +ModelId. Once the DSP firmware is updated, the GStreamer plugins can immediately wire it +in by setting the ``selected-model`` property to the new DSP firmware ID. + +However, if you need to extend the plugins themselves to better support a new model +category or add automatic detection logic, the following sections show the code you +would modify in the GStreamer plugin source files. Section :ref:`pub_edgeai_am62dx_building_plugin` +shows how to build the new modified plugin. + +Adding a New GStreamer Property to ``tidspkernel`` +-------------------------------------------------- + +If your new model requires an additional configuration parameter (for example, a +``model-specific-param`` flag), add it to the plugin header and implementation: + +**1. Header: ``gstdspkernel.h`` - Add to the struct:** + +.. code-block:: c + + struct _GstDspKernel + { + /* ... existing properties ... */ + guint fft_size; + guint window_frames; + guint selected_model; + guint model_elems; + + /* NEW: Add your custom property */ + guint model_specific_param; /* New parameter for custom model */ + }; + +**2. Implementation: ``gstdspkernel.cpp`` - Add property enum:** + +.. code-block:: c + + /* Property IDs */ + enum + { + PROP_0, + PROP_MSG_TYPE, + PROP_INTERLEAVE_DIRECTION, + PROP_HOP_SIZE, + PROP_FFT_SIZE, + PROP_WINDOW_FRAMES, + PROP_BATCH_SIZE, + PROP_SELECTED_MODEL, + PROP_MODEL_ELEMS, + PROP_MODEL_PATH, + PROP_OVERLAP_FRAMES, + PROP_CHUNKING_MODE, + PROP_MAX_STREAM_SAMPLES, + + /* NEW: Add your property ID */ + PROP_MODEL_SPECIFIC_PARAM + }; + +**3. Register the property in ``class_init()``:** + +.. code-block:: c + + g_object_class_install_property (gobject_class, PROP_MODEL_SPECIFIC_PARAM, + g_param_spec_uint ("model-specific-param", "Model Specific Param", + "Custom parameter for new model architecture", + 0, 255, 0, /* min, max, default */ + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + +**4. Add getter/setter in ``set_property()`` and ``get_property()``:** + +.. code-block:: c + + static void + gst_dsp_kernel_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) + { + GstDspKernel *kernel = GST_DSP_KERNEL (object); + + switch (prop_id) { + /* ... existing properties ... */ + case PROP_MODEL_SPECIFIC_PARAM: + kernel->model_specific_param = g_value_get_uint (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + } + + static void + gst_dsp_kernel_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) + { + GstDspKernel *kernel = GST_DSP_KERNEL (object); + + switch (prop_id) { + /* ... existing properties ... */ + case PROP_MODEL_SPECIFIC_PARAM: + g_value_set_uint (value, kernel->model_specific_param); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + } + +**5. Verify the property is registered (after rebuild and install):** + +After rebuilding and installing the plugin, verify that the new property appears in the +element's property list using ``gst-inspect-1.0``: + +.. code-block:: bash + + root@am62dxx-evm:~# gst-inspect-1.0 tidspkernel | grep -A 5 "model-specific-param" + +Expected output: + +.. code-block:: text + + model-specific-param : Custom parameter for new model architecture + flags: readable, writable + Unsigned Integer. Range: 0 - 255 Default: 0 + +If your new property appears in the output with the correct type, range, and default value, +the registration was successful. You can now use it in GStreamer pipelines: + +.. code-block:: bash + + gst-launch-1.0 -v ... tidspkernel name=stft ... model-specific-param=1 ... ! ... + +Adding a New Property to ``titvm`` +---------------------------------- + +Similarly, for the TVM inference element, if you need a new parameter (for example, +``inference-threshold`` for filtering low-confidence predictions): + +**1. Header: ``gsttitvm.h`` - Add to struct:** + +.. code-block:: c + + struct _GstTiTvm + { + /* ... existing properties ... */ + gchar *model_path; + gchar *class_map_path; + guint top_k; + guint daemon_timeout_ms; + + /* NEW: Add your custom property */ + gfloat inference_threshold; /* Confidence threshold (0.0-1.0) */ + }; + +**2. Implementation: ``gsttitvm.cpp`` - Add property enum and register:** + +.. code-block:: c + + enum + { + PROP_0, + PROP_MODEL_PATH, + PROP_CLASS_MAP_PATH, + PROP_TOP_K, + PROP_DAEMON_TIMEOUT_MS, + + /* NEW: Add your property ID */ + PROP_INFERENCE_THRESHOLD + }; + + /* In class_init() */ + g_object_class_install_property (gobject_class, PROP_INFERENCE_THRESHOLD, + g_param_spec_float ("inference-threshold", "Inference Threshold", + "Minimum confidence score to report predictions (0.0-1.0)", + 0.0f, 1.0f, 0.5f, /* min, max, default */ + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + +**3. Use in inference post-processing:** + +.. code-block:: c + + /* After getting inference output from TVM, filter by threshold */ + for (guint i = 0; i < output_size; ++i) { + gfloat confidence = output_buffer[i]; + + if (confidence >= tvm->inference_threshold) { + GST_DEBUG_OBJECT (tvm, "Prediction %u: confidence=%.3f (above threshold %.3f)", + i, confidence, tvm->inference_threshold); + } + } + +Automatic Model Detection from Path +----------------------------------- + +If you want the plugins to auto-detect your new model based on its directory path name, +add a mapping in the model detection logic in ``gstdspkernel.cpp``: + +.. code-block:: c + + struct { + const gchar *path_substring; + guint selected_model; + guint model_elems; + guint fft_size; + guint hop_size; + guint window_frames; + } known_models[] = { + { "gcrn", 2, 322, 320, 160, 401 }, + { "gtcrn", 1, 322, 320, 160, 401 }, + { "dccrn", 0, 322, 320, 160, 401 }, + { "yamnet", 4, 64, 512, 160, 96 }, + { "vggish", 3, 128, 512, 160, 96 }, + + /* NEW: Add your model */ + { "my_custom_model", 5, 256, 512, 256, 100 }, + + { NULL, 0, 0, 0, 0, 0 } /* Sentinel */ + }; + +The ``gst_dsp_kernel_detect_model()`` function iterates this table and auto-sets +``selected-model``, ``model-elems``, ``fft-size``, ``hop-size``, and ``window-frames`` +when ``model-path`` contains the matching substring. + +.. _pub_edgeai_am62dx_building_plugin: + +Building the Modified Plugins +----------------------------- + +Plugin source files are located in the :file:`/opt/edgeai-gst-plugins/` repository. + +**Source File Locations:** + +- ``ext/ti/gstdspkernel.h`` — Header for DSP kernel element (STFT/ISTFT/deinterleave) +- ``ext/ti/gstdspkernel.cpp`` — Implementation of DSP kernel element +- ``ext/ti/gsttitvm.h`` — Header for TVM inference element +- ``ext/ti/gsttitvm.cpp`` — Implementation of TVM inference element +- ``utility/gsttirpmsgctx.h`` / ``utility/gsttirpmsgctx.cpp`` — RPMsg context utilities + +After modifying any of these files, rebuild the plugin. The meson build system automatically +detects all source files in ``ext/ti/`` — **no meson.build file changes are required**. + +Native Build on the Board +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + root@am62dxx-evm:/opt/edgeai-gst-plugins# export SOC=am62d + root@am62dxx-evm:/opt/edgeai-gst-plugins# meson setup build --prefix=/usr \ + -Dpkg_config_path=pkgconfig \ + -Dtvm-plugins=enabled \ + -Dstft-plugins=enabled \ + -Ddl-plugins=disabled \ + -Denable-tidl=disabled + root@am62dxx-evm:/opt/edgeai-gst-plugins# ninja -C build + root@am62dxx-evm:/opt/edgeai-gst-plugins# sudo ninja -C build install + root@am62dxx-evm:/opt/edgeai-gst-plugins# sudo ldconfig + +The updated plugin will be installed to :file:`/usr/lib/gstreamer-1.0/libgstti.so`. diff --git a/source/devices/AM62DX/edgeai/linux_audio_inference_app.rst b/source/devices/AM62DX/edgeai/linux_audio_inference_app.rst new file mode 100644 index 000000000..456a0bdf1 --- /dev/null +++ b/source/devices/AM62DX/edgeai/linux_audio_inference_app.rst @@ -0,0 +1,365 @@ +.. _am62dx_linux_audio_inference_app: + +######################### +C++ Inference Application +######################### + +Introduction +============ + +The AM62DX audio inference application (``rpmsg_inference_example``) is a C++ application +for running configurable audio and tensor pipelines across Linux and the C7x™ DSP. It is part +of the `rpmsg-dma `__ repository. The +application enables: + +- **Real-time Audio Processing** — Process live input or WAV file inputs +- **DSP Offload** — Offload signal chains (ex: STFT, ISTFT and deinterleave/interleave) to the C7x™ DSP over RPMsg-DMA +- **TVM Inference** — Run TVM-compiled deep learning models on the C7x™ NPU +- **Flexible I/O** — WAV files, live ALSA capture, raw PCM streams, or binary tensors +- **JSON Pipeline Configuration** — Define processing chains via JSON files +- **Model Daemon Mode** — A model daemon hot-reloads artifacts without an application rebuild + +Supported Pipelines +=================== + +.. list-table:: + :header-rows: 1 + :widths: 30 30 20 20 + + * - Pipeline + - JSON file + - Default input + - Output + * - Speech enhancement (GCRN) + - :file:`pipeline_speech_enhancement.json` + - :file:`input_audio.wav` + - Enhanced WAV file + * - Audio classification (YAMNet) + - :file:`pipeline_audio_classification_yamnet.json` + - :file:`speech_whistling2.wav` + - Top-k class names and scores + * - Audio classification (VGGish) + - :file:`pipeline_audio_classification_vggish.json` + - :file:`urbansound_16k.wav` + - Top-k class names and scores + +Prerequisites +============= + +All runtime libraries, model artifacts, sample audio files, model daemon, and DSP firmware +verification are pre-packaged in the AM62DX EdgeAI image. See +:ref:`Models, Artifacts and Setup ` for details. + +To build the application from source, the following host tools are required: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Tool + - Purpose + * - ARM Clang Toolchain + - From Processor SDK Linux devkit; for cross-compilation + * - CMake 3.10 or higher + - Build system + * - ``make``, ``git`` + - Build utilities + * - Apache TVM headers (``TVM_ROOT``) + - Refer to `TI TVM Documentation `__ + +********************* +Building on AM62D EVM +********************* + +The application is pre-installed on the AM62DX EdgeAI image. If you need to rebuild it +natively on the AM62D EVM, clone the source and build: + +.. code-block:: console + + root@am62dxx-evm:~# git clone https://github.com/TexasInstruments/rpmsg-dma.git -b main + root@am62dxx-evm:~# cd rpmsg-dma + +The example is under ``example/edge-ai/`` and contains: + +.. code-block:: text + + example/edge-ai/ + ├── CMakeLists.txt + ├── src/ application source files + ├── include/ header files + ├── json_files/ pipeline JSON configurations + ├── labels/ classification label files + ├── input_audio/ sample WAV input files + ├── tvm-model-daemon.service + └── tvm-model-preload.service + +.. code-block:: console + + root@am62dxx-evm:~/rpmsg-dma# cmake -S . -B build -DBUILD_EDGE_AI_EXAMPLE=ON \ + -DBUILD_AUDIO_OFFLOAD_EXAMPLE=OFF \ + -DBUILD_2DFFT_OFFLOAD_EXAMPLE=OFF \ + -DBUILD_SIGNAL_CHAIN_BIQUAD_EXAMPLE=OFF + root@am62dxx-evm:~/rpmsg-dma# cmake --build build + +**************** +Building on Host +**************** + +#. Clone the source repository: + + .. code-block:: console + + host# git clone https://github.com/TexasInstruments/rpmsg-dma.git -b main + host# cd rpmsg-dma + +#. Download and install the AM62DX Linux SDK from |__SDK_DOWNLOAD_URL__|. + Extract the SDK tarball and run the installation program on your development host. + +#. Prepare the environment for cross compilation: + + .. code-block:: console + + host# source /linux-devkit/environment-setup + +#. Set the Neo-TVM path: + + .. code-block:: console + + host# export NEO_TVM_PATH=/path/to/neo-tvm + +#. Build only the edge-ai example: + + .. code-block:: console + + [linux-devkit]:> cmake -S . -B build -DBUILD_EDGE_AI_EXAMPLE=ON \ + -DBUILD_AUDIO_OFFLOAD_EXAMPLE=OFF \ + -DBUILD_2DFFT_OFFLOAD_EXAMPLE=OFF \ + -DBUILD_SIGNAL_CHAIN_BIQUAD_EXAMPLE=OFF + [linux-devkit]:> cmake --build build + +#. The following binaries are generated after a successful build: + + - ``build/example/edge-ai/rpmsg_inference_example`` — main inference application + - ``build/example/edge-ai/tvm_model_daemon`` — persistent model daemon + + Transfer the binaries to the AM62D EVM: + + .. code-block:: console + + host# scp build/example/edge-ai/rpmsg_inference_example root@:/usr/bin/ + host# scp build/example/edge-ai/tvm_model_daemon root@:/usr/bin/ + + Or copy directly to a mounted SD card: + + .. code-block:: console + + host# cp build/example/edge-ai/rpmsg_inference_example /mnt/rootfs/usr/bin/ + host# cp build/example/edge-ai/tvm_model_daemon /mnt/rootfs/usr/bin/ + +Basic Syntax +============ + +.. code-block:: console + + rpmsg_inference_example # file input from JSON + rpmsg_inference_example --device # live ALSA capture + rpmsg_inference_example --input-file # override input file + rpmsg_inference_example --stream # raw PCM from stdin + rpmsg_inference_example --debug # enable per-batch logs + rpmsg_inference_example --preload # preload default model + rpmsg_inference_example --version + rpmsg_inference_example --help + +***************************** +GCRN based Speech Enhancement +***************************** + +Speech enhancement removes background noise from input audio source while keeping the speech +intelligible — useful for voice calls, meeting recordings, hearables, and any product +that captures speech in a noisy environment. + +Speech Enhancement Pipeline using GCRN +-------------------------------------- + +Run with the default WAV file input: + +.. code-block:: console + + am62dxx-evm:~# rpmsg_inference_example /usr/share/tvm_inference/json/pipeline_speech_enhancement.json + edge-ai v0.0.4 (built Sep 21 2026 10:14:19) + =========================================== + RPMsg Inference Example + =========================================== + + [App] Pipeline type: speech_enhancement + [App] Description: GCRN 3-stage speech pipeline: STFT analyze -> TVM inference -> ISTFT synthesize + [App] TVM artifacts configured: /usr/share/tvm_inference/artifacts/gcrn + [App] Model changed (/usr/share/tvm_inference/artifacts/yamnet -> /usr/share/tvm_inference/artifacts/gcrn), requesting daemon reload... + [App] Daemon reloaded model: /usr/share/tvm_inference/artifacts/gcrn + [App] Input configured: /usr/share/tvm_inference/input/input_audio.wav + [App] Stages: 5 + [App] STFT parameters: HOP=160 MODEL_ELEMS=322 BATCH_N=64 TOTAL_FRAMES=401 + [App] ISTFT parameters: HOP=160 MODEL_ELEMS=322 BATCH_N=64 TOTAL_FRAMES=401 + [App] STFT spectral buffer: 516488 bytes + [App] ISTFT spectral buffer: 516488 bytes + [App] Audio file info: 156302 samples, 16000Hz, 1 channel(s) + [App] Loaded 156302 audio samples (9.76887 seconds) + eptdev = rpmsg-char-8-1303 + Created endpt device rpmsg-char-8-1303, fd = 4 port = 1027 + [App] DMA buffers (reused across sequential stages): + [App] A (PCM / planar): phys=0xc0600000 size=516488 + [App] B (interleaved): phys=0xc0680000 size=516488 + [App] Overlap-save chunking: + [App] TOTAL_FRAMES=401 OVERLAP_FRAMES=100 HOP_FRAMES=301 n_chunks=3 + [TVM] Connected to model daemon at /var/run/tvm-inference.sock + [App] TVM initialized, input shape [1,2,401,161] + [App] Chunk 1/3 [keep samples 0..56160] | STFT=17.3ms TVM=6782.4ms ISTFT=18.4ms total=6821.0ms + [App] Chunk 2/3 [keep samples 8000..56160] | STFT=16.3ms TVM=6781.0ms ISTFT=17.7ms total=6817.6ms + [App] Chunk 3/3 [keep samples 8000..59982] | STFT=16.3ms TVM=6780.0ms ISTFT=18.2ms total=6817.3ms + [App] All chunks done | total=20456.2ms | 976 output frames (156302 samples) + [App] Saving audio to: processed_output.wav + [App] Output file info: 156302 samples, 16000Hz, 1 channel(s) + [App] Successfully saved 156302 samples to processed_output.wav + [App] Duration: 9.8 seconds + [App] Saved to processed_output.wav + PASSED + +****************************************** +YAMNet & VGGish based Audio Classification +****************************************** + +Audio classification identifies *what* sound is present in input audio source — a siren, +glass breaking, a dog barking, a machine running. YAMNet recognizes 521 general sound +classes, while VGGish is a smaller 10-class model. Pick YAMNet for broad coverage, or +VGGish when you need a lighter model for a fixed set of sounds. + +Audio Classification pipeline using YAMNet +------------------------------------------ + +Run with the default WAV file input: + +.. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example /usr/share/tvm_inference/json/pipeline_audio_classification_yamnet.json + edge-ai v0.0.4 (built Sep 21 2026 10:14:19) + =========================================== + RPMsg Inference Example + =========================================== + + [App] Pipeline type: audio_classification + [App] Description: YAMNet: speech -> STFT log-mel spectrogram -> TVM classification (521 classes) + [App] TVM artifacts configured: /usr/share/tvm_inference/artifacts/yamnet + [App] Model changed (/usr/share/tvm_inference/artifacts/vggish -> /usr/share/tvm_inference/artifacts/yamnet), requesting daemon reload... + [App] Daemon reloaded model: /usr/share/tvm_inference/artifacts/yamnet + [App] Input configured: /usr/share/tvm_inference/input/speech_whistling2.wav + [App] Stages: 2 + [App] STFT parameters: HOP=160 MODEL_ELEMS=64 BATCH_N=64 TOTAL_FRAMES=96 + [TVM] Connected to model daemon at /var/run/tvm-inference.sock + [App] TVM initialized, input shape [1,1,96,64] + [App] Loaded 521 class labels from /usr/share/tvm_inference/labels/yamnet_label_list.txt + [App] Audio file info: 78720 samples, 16000Hz, 1 channel(s) + [App] Loaded 78720 audio samples (4.92 seconds) + eptdev = rpmsg-char-8-1298 + Created endpt device rpmsg-char-8-1298, fd = 5 port = 1027 + [App] DMA buffers: + [App] audio in: phys=0xc01b8000 size=20480 + [App] mel out: phys=0xc01d8000 size=24576 + [App] Total frames: 492 | Full chunks: 5 + [App] Chunk 1/5 STFT=21.9ms TVM=26.1ms total=48.0ms + [App] Chunk 1 top-5: + [App] 1. 4.7947 Speech + [App] 2. -2.6637 Narration, monologue + [App] 3. -3.7292 Speech synthesizer + [App] 4. -3.7292 Conversation + [App] 5. -4.7947 Inside, small room + [App] Chunk 2/5 STFT=9.4ms TVM=25.8ms total=35.2ms + [App] Chunk 2 top-5: + [App] 1. -1.5982 Animal + [App] 2. -1.5982 Bird + [App] 3. -1.5982 Whistling + [App] 4. -2.1310 Wild animals + [App] 5. -2.6637 Domestic animals, pets + [App] Chunk 3/5 STFT=9.4ms TVM=26.0ms total=35.4ms + [App] Chunk 3 top-5: + [App] 1. 3.1965 Whistling + [App] 2. -3.1965 Whistle + [App] 3. -3.7292 Alarm + [App] 4. -5.3275 Inside, small room + [App] 5. -5.8602 Bird + [App] Chunk 4/5 STFT=9.4ms TVM=26.3ms total=35.7ms + [App] Chunk 4 top-5: + [App] 1. 0.0000 Whistling + [App] 2. -1.0655 Whistle + [App] 3. -1.5982 Alarm + [App] 4. -5.8602 Inside, small room + [App] 5. -6.9257 Water + [App] Chunk 5/5 STFT=9.4ms TVM=26.0ms total=35.4ms + [App] Chunk 5 top-5: + [App] 1. 2.6637 Whistle + [App] 2. 1.0655 Alarm + [App] 3. 0.5327 Whistling + [App] 4. -5.3275 Inside, small room + [App] 5. -5.8602 Steam whistle + [App] Done: 492 mel frames | 5 TVM chunks | 190.3ms + PASSED + [App] Application exited with code 0 + +Live ALSA device input: + +.. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example \ + /usr/share/tvm_inference/json/pipeline_audio_classification_yamnet.json \ + --device plughw:0,0 + +Stream raw PCM from stdin (mono, S16_LE, 16 kHz): + +.. code-block:: console + + root@am62dxx-evm:~# arecord -t raw -f S16_LE -c 1 -r 16000 | \ + rpmsg_inference_example \ + /usr/share/tvm_inference/json/pipeline_audio_classification_yamnet.json \ + --stream + +Audio Classification Pipeline using VGGish +------------------------------------------ + +Run with the default WAV file input: + +.. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example /usr/share/tvm_inference/json/pipeline_audio_classification_vggish.json + edge-ai v0.0.4 (built Sep 21 2026 10:14:19) + =========================================== + RPMsg Inference Example + =========================================== + + [App] Pipeline type: audio_classification + [App] Description: VGGish: audio -> STFT log-mel spectrogram -> TVM classification (UrbanSound8K 10 classes) + [App] TVM artifacts configured: /usr/share/tvm_inference/artifacts/vggish + [App] Model changed (/usr/share/tvm_inference/artifacts/gcrn -> /usr/share/tvm_inference/artifacts/vggish), requesting daemon reload... + [App] Daemon reloaded model: /usr/share/tvm_inference/artifacts/vggish + [App] Input configured: /usr/share/tvm_inference/input/urbansound_16k.wav + [App] Stages: 2 + [App] STFT parameters: HOP=512 MODEL_ELEMS=64 BATCH_N=64 TOTAL_FRAMES=126 + [TVM] Connected to model daemon at /var/run/tvm-inference.sock + [App] TVM initialized, input shape [1,1,126,64] + [App] Loaded 10 class labels from /usr/share/tvm_inference/labels/vggish_label_list.txt + [App] Audio file info: 128000 samples, 16000Hz, 1 channel(s) + [App] Loaded 128000 audio samples (8 seconds) + eptdev = rpmsg-char-8-1290 + Created endpt device rpmsg-char-8-1290, fd = 5 port = 1027 + [App] DMA buffers: + [App] audio in: phys=0xc02b0000 size=65536 + [App] mel out: phys=0xc01b8000 size=32256 + [App] Total frames: 250 | Full chunks: 1 + [App] Chunk 1/1 STFT=44.4ms TVM=31.1ms total=75.5ms + [App] Chunk 1 top-5: + [App] 1. 8.4112 Street music + [App] 2. 8.2060 Children playing + [App] 3. -4.5133 Air conditioner + [App] 4. -5.1288 Dog bark + [App] 5. -5.5391 Car horn + [App] Done: 250 mel frames | 1 TVM chunks | 75.6ms + PASSED + [App] Application exited with code 0 diff --git a/source/devices/AM62DX/edgeai/model_compile_evaluate.rst b/source/devices/AM62DX/edgeai/model_compile_evaluate.rst new file mode 100644 index 000000000..97f75da47 --- /dev/null +++ b/source/devices/AM62DX/edgeai/model_compile_evaluate.rst @@ -0,0 +1,251 @@ +.. include:: + +.. _edgeai_compile_evaluate: + +########################## +Model Compile and Evaluate +########################## + +The :ref:`EdgeAI Model Zoo ` runs *pre-compiled* artifacts +on the |__PART_FAMILY_NAME__| target. This section covers the **x86 host +workflow** that produces those artifacts: how to compile an audio model for +the C7\ |trade| NPU, run host-emulation inference, and measure accuracy against a +data set - using **edgeai-tidlrunner**. It runs entirely on an Ubuntu x86 PC; +nothing here touches the target. + +This is the workflow for *any* audio model, not just the reference models. The +four reference models below are worked examples. To bring your own model you +point the same ``tidlrunner-cli`` at your ONNX graph and a per-model YAML config, +using a reference config as a starting template. Which operators offload to the +C7\ |trade| NPU (versus fall back to the Arm core) is determined by TIDL - see +`edgeai-tidl-tools `__ for +the authoritative list of supported operators. + +***************** +edgeai-tidlrunner +***************** + +edgeai-tidlrunner is a bring-your-own-model (BYOM) command-line wrapper over the +core `edgeai-tidl-tools `__ +compilation stack. A single YAML config per model drives every stage, and one +CLI - ``tidlrunner-cli`` - exposes the stages this section uses: + +.. list-table:: + :header-rows: 1 + :widths: 18 82 + + * - Command + - What it does + * - ``compile`` + - Imports the ONNX graph, quantizes and partitions it, and writes the + deployable TIDL/TVM-RT artifacts. + * - ``infer`` + - Runs the compiled model on the x86 host (PC emulation) to check output + correctness before deploying to the target. + * - ``evaluate`` + - Runs the compiled model over a data set and reports accuracy or + speech-enhancement metrics. + * - ``inspect`` + - One-shot compile + inference + reference comparison that emits an + interactive HTML analysis report (see :ref:`Model Inspect `). + +All commands are run from the repository root (``edgeai-tidlrunner/``) and take +``--config_path `` pointing at the per-model YAML. The same config file is +reused across ``compile``, ``infer``, and ``evaluate``. + +.. _edgeai-am62d-setup: + +***** +Setup +***** + +Model compilation runs on an x86 PC (Ubuntu Linux recommended). The AM62D audio +flow is self-contained: a single setup script installs the whole TVM toolchain +into a dedicated Python 3.10 environment. The TIDL tools target Python 3.10. +These steps assume `pyenv `__. + +#. **Clone the repository** (https://github.com/TexasInstruments/edgeai-tidlrunner): + + .. code-block:: console + + $ git clone https://github.com/TexasInstruments/edgeai-tidlrunner.git + $ cd edgeai-tidlrunner + +#. **Create and activate a dedicated Python 3.10 environment.** The AM62D flow + uses a TI TVM wheel; keep it in its own venv (``tidlrunner-am62d``) + so it never clobbers a standard ``tidlrunner`` setup: + + .. code-block:: console + + $ pyenv install 3.10 + $ pyenv virtualenv 3.10 tidlrunner-am62d + $ pyenv activate tidlrunner-am62d + +#. **Run the AM62D setup script.** With the venv active, one script installs + everything this flow needs: + + .. code-block:: console + + $ ./devices/setup_am62d.sh + + It downloads the ARM GCC 15.2 and C7000 CGT 5.0.0 LTS cross-toolchains into + ``tools/tidl_tools_package/bin/`` (skipped if already present), installs the + x86 TI TVM wheel - which bundles the AM62D x86 TIDL tools *inside* + the package - and installs ``tidlrunner[pc,audio]``, ``tools``, ``onnxruntime``, + and ``tidl_onnx_model_optimizer``. The ``[audio]`` extra pulls in the audio + packages (``librosa``, ``soundfile``, ``scipy``, ``pesq``, ``pystoi``, + ``scikit-learn``). No separate ``pip install`` is needed. + +#. **Source the env script once per shell.** TVM ``dlopen``\ s the TIDL runtime + (:file:`libvx_tidl_rt.so`) before Python's own import runs, so the tool paths must be + live in the shell *before* ``tidlrunner-cli`` starts. Source (do **not** execute) + :file:`devices/am62d_env.sh` once; every ``tidlrunner-cli`` command run afterward in + that shell picks the vars up: + + .. code-block:: console + + $ source devices/am62d_env.sh + + It exports: + + .. list-table:: + :header-rows: 1 + :widths: 26 74 + + * - Variable + - Value + * - ``TIDL_TOOLS_PATH`` + - ``/3rdparty/x86_tidl_tools/AM62D`` (resolved from the + active venv's installed ``tvm``) + * - ``LD_LIBRARY_PATH`` + - prepends ``TIDL_TOOLS_PATH`` so :file:`libvx_tidl_rt.so` resolves + * - ``ARM64_GCC_PATH`` + - the aarch64 GNU toolchain installed by :file:`setup_am62d.sh` under + :file:`tools/tidl_tools_package/bin/` (override by exporting before sourcing) + * - ``CGT7X_ROOT`` + - the C7000 code-generation tools installed by :file:`setup_am62d.sh` under + :file:`tools/tidl_tools_package/bin/` (override by exporting before sourcing) + * - ``SOC`` + - ``am62d`` + +Data Sets +========= + +Accuracy evaluation needs the reference data sets. Compilation alone can run on +random data (for latency-only measurement), but ``evaluate`` requires the real +sets. Download them from the repository root: + +.. code-block:: console + + $ # UrbanSound8K (~5.6 GB) - sound classification (VGGish11, YAMNet) + $ bash examples/audio/scripts/download_urbansound8k.sh + + $ # VoiceBank-DEMAND-16k (~2 GB) - speech enhancement (GCRN, GTCRN) + $ python3 examples/audio/scripts/download_voicebank_demand.py + +They land under ``data/datasets/UrbanSound8K/`` and +``data/datasets/VoiceBank-DEMAND-16k/`` - the paths the model configs expect. +The classification configs use UrbanSound8K **fold 10** (837 samples) as the +test set; the speech-enhancement configs use the VoiceBank-DEMAND-16k **test** set +(824 files). + +The reference ONNX models auto-download at compile time via ``.link`` files, so +fetching them ahead of time is **optional**. To pre-download all reference ONNX models: + +.. code-block:: console + + $ bash examples/audio/scripts/download_audio_models.sh + +******************** +Compile and evaluate +******************** + +Each model has one config that drives all stages. All four run under the +``tidlrunner-am62d`` environment with :file:`devices/am62d_env.sh` already sourced +(see `Setup`_). Three models compile to the C7\ |trade| NPU through the TVM runtime; +GTCRN runs ARM-only through ONNX Runtime (no C7\ |trade| NPU offload) but uses the +same environment. The AM62D config paths are: + +.. list-table:: + :header-rows: 1 + :widths: 12 10 12 18 54 + + * - Model + - Model ID + - Runtime + - Tensor Bits + - Config path (relative to ``edgeai-tidlrunner/``) + * - VGGish11 + - ``ac-0101`` + - TVM-RT + - 8 + - :file:`data/configs/samples/models/audio/audio_classification/urbansound8k/vggish11_tvmrt_config.yaml` + * - YAMNet + - ``ac-0201`` + - TVM-RT + - 8 + - :file:`data/configs/samples/models/audio/audio_classification/urbansound8k/yamnet_tvmrt_config.yaml` + * - GCRN + - ``ase-0201`` + - TVM-RT + - 16 (LSTM), FP32 (Enc/Dec) + - :file:`data/configs/samples/models/audio/speech_enhancement/voicebank_demand_16k/gcrn_fixed_4sec_tvmrt_config.yaml` + * - GTCRN + - ``ase-0100`` + - ONNX-RT + - FP32 + - :file:`data/configs/samples/models/audio/speech_enhancement/voicebank_demand_16k/gtcrn_dns3_config.yaml` + +VGGish11 (TVM-RT, 8-bit) +======================== + +Runs on the C7\ |trade| NPU (``tidl_offload: true``): + +.. code-block:: console + + $ CFG=data/configs/samples/models/audio/audio_classification/urbansound8k/vggish11_tvmrt_config.yaml + $ tidlrunner-cli compile --config_path $CFG + $ tidlrunner-cli infer --config_path $CFG + $ tidlrunner-cli evaluate --config_path $CFG + +``evaluate`` reports top-1 / top-5 / macro-F1 over UrbanSound8K fold 10. + +YAMNet (TVM-RT, 8-bit) +====================== + +.. code-block:: console + + $ CFG=data/configs/samples/models/audio/audio_classification/urbansound8k/yamnet_tvmrt_config.yaml + $ tidlrunner-cli compile --config_path $CFG + $ tidlrunner-cli infer --config_path $CFG + $ tidlrunner-cli evaluate --config_path $CFG + +GCRN (TVM-RT, 16-bit) +===================== + +GCRN is C7\ |trade| NPU-offloaded at 16-bit (``tidl_offload: true``, fixed 4-second +input): + +.. code-block:: console + + $ CFG=data/configs/samples/models/audio/speech_enhancement/voicebank_demand_16k/gcrn_fixed_4sec_tvmrt_config.yaml + $ tidlrunner-cli compile --config_path $CFG + $ tidlrunner-cli infer --config_path $CFG + $ tidlrunner-cli evaluate --config_path $CFG + +``evaluate`` reports PESQ / STOI / SI-SDR over the VoiceBank-DEMAND-16k test set. + +GTCRN (ONNX-RT, ARM-only) +========================= + +GTCRN has a dynamic time axis and TIDL-unsupported operators, so it runs on the +Arm Cortex-A cores through ONNX Runtime (``tidl_offload: false``) - no C7\ |trade| NPU +offload. It uses the same ``tidlrunner-am62d`` environment as the other models; +it simply does not offload to the DSP: + +.. code-block:: console + + $ CFG=data/configs/samples/models/audio/speech_enhancement/voicebank_demand_16k/gtcrn_dns3_config.yaml + $ tidlrunner-cli compile --config_path $CFG + $ tidlrunner-cli infer --config_path $CFG + $ tidlrunner-cli evaluate --config_path $CFG diff --git a/source/devices/AM62DX/edgeai/model_inspect.rst b/source/devices/AM62DX/edgeai/model_inspect.rst new file mode 100644 index 000000000..07adad861 --- /dev/null +++ b/source/devices/AM62DX/edgeai/model_inspect.rst @@ -0,0 +1,65 @@ +.. _edgeai_model_inspect: + +############# +Model Inspect +############# + +Model Inspector analyzes a model on the x86 host after (or instead of) the +:ref:`compile and evaluate ` flow. It uses the same +edgeai-tidlrunner installation and environment described in +:ref:`Setup `. + +*************** +Model Inspector +*************** + +``tidlrunner-cli inspect`` is a one-shot analysis pipeline: it compiles the +model for TIDL, runs inference, re-runs a non-TIDL reference for comparison, +computes per-layer signal-to-noise ratios, and emits a **self-contained +interactive HTML report** - no need to run ``compile`` and ``infer`` +separately. The report opens in any browser and lands at: + +.. code-block:: text + + work_dirs/compile////inspect/modelinspector.html + +```` is the model's work directory, named after its model ID, +runtime, task, data set, and model, e.g. +``ase-0201_tvmrt_speech_enhancement_voicebank_demand_16k_gcrn_fixed_4sec_onnx``. + +The report has three tabs: + +- **Model Summary** - acceleration ratio (layers on the C7x™ DSP compared to Arm + fallback), input/output specs, and an interactive ONNX graph. +- **Subgraphs** - the TIDL subgraph(s) offloaded to the DSP, with per-layer + identity and a quantization analysis (INT8 versus FP32 activation histograms and + scatter plots). +- **Performance** - *estimated* per-layer processing time, DSP cycle breakdown + (total / IO / kernel), memory usage (DDR / L2 / MSMC), and SNR metrics. + +.. note:: + + All Performance-tab numbers are **estimates** from the compiler, not + on-target measurements. Model Inspector requires TIDL tools **11.02 or + later** (earlier versions emit SVG subgraph files it cannot read). + +GCRN showcase - an offloaded DSP subgraph +========================================= + +Because GCRN now compiles with ``tidl_offload: true`` on AM62D, its inspector +report shows a **real C7x™ DSP subgraph** rather than an all-Arm fallback: +the Subgraphs tab lists a single offloaded subgraph (16-bit) with its layer +mapping, and the Performance tab breaks that subgraph down by cycles and memory. +Generate it with the AM62D env sourced (see :ref:`Setup `): + +.. code-block:: console + + $ CFG=data/configs/samples/models/audio/speech_enhancement/voicebank_demand_16k/gcrn_fixed_4sec_tvmrt_config.yaml + $ tidlrunner-cli inspect --config_path $CFG + +For contrast, running ``inspect`` on the GTCRN (ONNX-RT) config produces a +report with **no** DSP subgraph - every layer is Arm fallback - which is what a +non-offloaded model looks like in the inspector. GTCRN also runs with +``tidl_offload: false``, so its work directory uses a ``notidl`` segment in +place of the ```` shown in the path template above, e.g. +:file:`work_dirs/compile/AM62D/notidl//...`. diff --git a/source/devices/AM62DX/edgeai/model_zoo.rst b/source/devices/AM62DX/edgeai/model_zoo.rst new file mode 100644 index 000000000..c9c7d56d9 --- /dev/null +++ b/source/devices/AM62DX/edgeai/model_zoo.rst @@ -0,0 +1,314 @@ +.. include:: + +.. _edgeai_model_zoo: + +################ +EdgeAI Model Zoo +################ + +The EdgeAI ModelZoo delivers out-of-box audio deep-learning demos that run +directly on the |__PART_FAMILY_NAME__| target. Each reference model ships as an +`ONNX `__ graph plus, where applicable, a pre-compiled +TVM-RT + TIDL artifact, so you can reproduce these examples on the AM62D without +compiling anything yourself. Two demo front-ends are provided: standalone +Python inference scripts and Jupyter notebooks. The same on-target runtime hosts +your own compiled artifacts once you build them through the x86 flow. This +section runs entirely **on the target**; the x86 compile-and-evaluate flow is covered +in :ref:`Model Compile and Evaluate `. + +.. note:: + + The models in this repository are made available for experimentation and + development. They are not intended for deployment in production. + +********** +What ships +********** + +.. list-table:: + :header-rows: 1 + :widths: 18 26 14 14 28 + + * - Model + - Task + - Model ID + - On target + - Demo + * - VGGish11 + - Sound classification (UrbanSound8K) + - ``ac-0101`` + - C7\ |trade| NPU (TVM-RT + TIDL, 8-bit) + - Script + notebook + * - YAMNet + - Sound classification (UrbanSound8K) + - ``ac-0201`` + - C7\ |trade| NPU (TVM-RT + TIDL, 8-bit) + - Script + notebook + * - GCRN + - Speech enhancement (VoiceBank-DEMAND-16k) + - ``ase-0201`` + - C7\ |trade| NPU (TVM-RT + TIDL, 16-bit) + - Script + notebook + * - GTCRN + - Speech enhancement (VoiceBank-DEMAND-16k) + - ``ase-0100`` + - Arm core (ONNX runtime, FP32) + - Script + notebook + +VGGish11, YAMNet, and GCRN each ship a pre-compiled TVM-RT + TIDL artifact and +run offloaded to the C7\ |trade| NPU. GTCRN ships **no** compiled artifact - it +runs FP32 on the Arm core through the ONNX runtime, packaged as a plain ONNX +Runtime artifact. + +*************** +On-target setup +*************** + +The steps below run on the AM62D target (aarch64), on the Linux command line of +the Processor SDK rootfs. The notebooks and scripts run in a Python virtual +environment that reuses the TIDL-enabled TVM runtime already present in the +rootfs. + +#. **Clone the repository.** Clone the AudioAI ModelZoo repository + (https://github.com/TexasInstruments/audioai-modelzoo): + + .. code-block:: console + + $ mkdir -p ~/tidl && cd ~/tidl + $ git clone https://github.com/TexasInstruments/audioai-modelzoo.git + $ cd audioai-modelzoo + +#. **Download the model artifacts.** Downloads the three pre-compiled + TVM-RT + TIDL artifacts (VGGish11, YAMNet, GCRN) plus the GTCRN ONNX Runtime + artifact into ``model_artifacts//am62d/``: + + .. code-block:: console + + $ ./download_models.sh -y + + The script defaults to an interactive selection menu; the ``-y`` flag + downloads everything non-interactively. Add ``-l`` to print the source URLs + without downloading. + +#. **Create the virtual environment.** Creates the venv at + ~/venv/modelzoo with ``--system-site-packages`` so the TIDL-enabled + runtime from the rootfs is visible: + + .. code-block:: console + + $ ./setup_venv.sh + +#. **Activate the venv.** Activate it in every new shell before running the + demos: + + .. code-block:: console + + $ source ~/venv/modelzoo/bin/activate + +********************** +Inference script demos +********************** + +Run each script from its own model directory in the activated venv. All four +models ship a command-line script. + +YAMNet (sound classification) +============================= + +.. code-block:: console + + $ cd ~/tidl/audioai-modelzoo/inference/yamnet_sc + $ python3 yamnet_infer_audio.py --audio-file samples/miaow_16k.wav + +The script converts the audio into log-mel patches, runs each patch through +the TVM-RT + TIDL session, and averages the per-patch scores. For the bundled +:file:`miaow_16k.wav` clip the top classes are cat/animal sounds. Expected output +(abridged): + +.. code-block:: text + + ====================================================================== + YAMNET INFERENCE SUMMARY + ====================================================================== + + Audio File: miaow_16k.wav + Artifacts: ac-0201_tvmrt_audio_classification_urbansound8k_yamnet_onnx + Audio Duration: 6.73 seconds + Number of Patches: 7 + + ====================================================================== + TOP 10 PREDICTIONS (averaged across all patches): + ====================================================================== + Rank Score Class + ---------------------------------------------------------------------- + 1 ... Cat + 2 ... Animal + ... + ====================================================================== + PERFORMANCE METRICS: + ====================================================================== + Inference Time (all patches): 17.53 ms total | 7 patches + RTF = 0.0026 (0.018s / 6.73s) + ====================================================================== + +A second sample, :file:`samples/speech_whistling2.wav`, is also included. + +VGGish11 (sound classification) +=============================== + +.. code-block:: console + + $ cd ~/tidl/audioai-modelzoo/inference/vggish11_sc + $ python3 vggish_infer_audio.py --audio-file sample_wav/139951-9-0-9.wav + +The bundled clip is UrbanSound8K class 9 (street music). Expected +output (abridged): + +.. code-block:: text + + ====================================================================== + VGGISH11 INFERENCE SUMMARY + ====================================================================== + + Audio File: 139951-9-0-9.wav + Artifacts: ac-0101_tvmrt_audio_classification_urbansound8k_vggish11_onnx + Audio Duration: 4.00 seconds + Top Class: Street music (Class 9) - Score: ... + + Performance: + Inference: 8.88 ms + Throughput: RTF = 0.0022 (0.009s / 4.00s) + ====================================================================== + +GCRN (speech enhancement) +========================= + +.. code-block:: console + + $ cd ~/tidl/audioai-modelzoo/inference/gcrn_se + $ python3 gcrn_infer_audio.py --input sample_wav/noisy.wav + +The script writes the enhanced audio to :file:`enhanced.wav` beside the input and a +per-block timing report to :file:`gcrn-benchmark.md`. Pass ``--output`` to change +the destination wav. Expected output (abridged): + +.. code-block:: text + + Machine: aarch64 SOC: am62d TIDL_VER: ... + Loading TVM + TIDL session... + Running inference on noisy.wav (9.77s)... + Wrote enhanced audio to sample_wav/enhanced.wav + + GCRN TVM+TIDL benchmark (3 chunk(s), 9.77s audio) + + block total (ms) per-chunk (ms) + ... + Real-Time Factor (inference / audio) = 3.9500 + + Wrote benchmark report to sample_wav/gcrn-benchmark.md + +GTCRN (speech enhancement) +========================== + +.. code-block:: console + + $ cd ~/tidl/audioai-modelzoo/inference/gtcrn_se + $ python3 gtcrn_infer_audio.py --input sample_wav/mix.wav + +Unlike the other three models, GTCRN has no TVM-RT + TIDL artifact - the script +loads the ONNX graph directly and runs it on ``CPUExecutionProvider`` (Arm +core, FP32). It writes the enhanced audio to :file:`enh_onnx.wav` beside the input; +pass ``--output`` to change the destination wav. Expected output (abridged): + +.. code-block:: text + + Machine: aarch64 + Model: .../ase-0100_onnxrt_speech_enhancement_voicebank_demand_16k_gtcrn_dns3_onnx/model/gtcrn_dns3.onnx + Loading ONNX Runtime session... + Running inference on mix.wav (9.77s)... + Wrote enhanced audio to sample_wav/enh_onnx.wav + + GTCRN ONNX Runtime (CPU) benchmark (9.77s audio) + + block total (ms) + ------------------------ ---------- + pre-processing (STFT) ... + model inference ... + post-processing (iSTFT) ... + end-to-end ... + + Real-Time Factor (inference / audio) = 0.0700 + +GTCRN is already faster than real time on the Arm core, so it needs no NPU +offload. + +.. _edgeai_model_zoo_notebooks: + +********************** +Jupyter notebook demos +********************** + +All four reference models have a Jupyter notebook demo. Start Jupyter Lab from +the activated venv: + +.. code-block:: console + + $ cd ~/tidl/audioai-modelzoo + $ ./jupyter_lab_venv.sh + +The launcher prints a highlighted access URL. Open the URL in a browser on a +machine that can reach the target and enter the token ``tidl`` when prompted: + +.. code-block:: text + + Access URL: http://:8888/lab?token=tidl + +Jupyter Lab opens with the GCRN notebook loaded. To run a different model, pick +its notebook from the file browser in the left sidebar. The notebooks are: + +.. list-table:: + :header-rows: 1 + :widths: 22 40 + + * - Model + - Notebook + * - GCRN + - ``inference/gcrn_se/gcrn_inference.ipynb`` + * - GTCRN + - ``inference/gtcrn_se/gtcrn_inference.ipynb`` + * - VGGish11 + - ``inference/vggish11_sc/vggish_inference.ipynb`` + * - YAMNet + - ``inference/yamnet_sc/yamnet_inference.ipynb`` + +****************** +Compiled artifacts +****************** + +:file:`download_models.sh` places the artifacts under +:file:`model_artifacts//am62d/`. Each model gets its own directory, +named after its model ID, containing a ``model/`` folder (the trained ONNX +model) and, where applicable, an ``artifacts/`` folder (the compiled model +artifacts). VGGish11, YAMNet, and GCRN each get a pre-compiled TVM-RT + TIDL +artifact: + +- ``ac-0101`` - ``ac-0101_tvmrt_audio_classification_urbansound8k_vggish11_onnx`` (VGGish11) +- ``ac-0201`` - ``ac-0201_tvmrt_audio_classification_urbansound8k_yamnet_onnx`` (YAMNet) +- ``ase-0201`` - ``ase-0201_tvmrt_speech_enhancement_voicebank_demand_16k_gcrn_fixed_4sec_onnx`` (GCRN) + +Each of those artifact directories' :file:`artifacts/` folder holds the TVM-RT +deployment artifacts (:file:`deploy_lib.so`, :file:`deploy_graph.json`, :file:`deploy_param.params`) +in both target (``.evm``, aarch64) and host (``.pc``, x86) variants. The demo +scripts and notebooks select the ``.evm`` variant automatically on the target. + +GTCRN (``ase-0100``) gets an ONNX Runtime artifact instead of a +compiled one: + +- ``ase-0100`` - ``ase-0100_onnxrt_speech_enhancement_voicebank_demand_16k_gtcrn_dns3_onnx`` (GTCRN) + +Its directory holds only the ``model/`` folder, with the ``gtcrn_dns3.onnx`` +graph - there is no ``artifacts/`` folder since no TIDL compilation is +involved. The demo script and notebook load the ONNX model directly with +``CPUExecutionProvider`` and run FP32 on the Arm core. Compiled TVM-RT + TIDL +artifacts for your own models are produced by the x86 workflow in +:ref:`Model Compile and Evaluate `. diff --git a/source/devices/AM62DX/edgeai/models_artifacts_and_setup.rst b/source/devices/AM62DX/edgeai/models_artifacts_and_setup.rst new file mode 100644 index 000000000..bc6b01276 --- /dev/null +++ b/source/devices/AM62DX/edgeai/models_artifacts_and_setup.rst @@ -0,0 +1,180 @@ +.. _am62dx_edgeai_models_artifacts_and_setup: + +######################### +Model Artifacts and Setup +######################### + +The AM62DX EdgeAI image includes three pre-compiled, pre-packaged models ready to run +out of the box — no compilation required. These models, artifacts, and setup steps are +common to both the Linux inference application and the GStreamer pipelines. + +******************* +Pre-Packaged Models +******************* + +.. list-table:: + :header-rows: 1 + :widths: 15 20 10 25 30 + + * - Model + - Type + - ModelId + - Artifacts path + - Description + * - GCRN + - Speech enhancement + - 2 + - :file:`/usr/share/tvm_inference/artifacts/gcrn/` + - Removes background noise from speech; produces enhanced WAV output + * - YAMNet + - Audio classification + - 4 + - :file:`/usr/share/tvm_inference/artifacts/yamnet/` + - 521-class AudioSet audio classification + * - VGGish + - Audio classification + - 3 + - :file:`/usr/share/tvm_inference/artifacts/vggish/` + - 10-class UrbanSound8K audio classification + +*************** +Model Artifacts +*************** + +Each model directory under :file:`/usr/share/tvm_inference/artifacts//` contains: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - File + - Description + * - :file:`deploy_graph.json` + - Graph description; defines the neural network topology and input/output shapes + * - :file:`deploy_lib.so` + - Compiled model library; shared object for TVM runtime + * - :file:`deploy_param.params` + - Model weights; binary parameter data + +*********** +Label Files +*********** + +Classification models ship with label files under :file:`/usr/share/tvm_inference/labels/`: + +.. list-table:: + :header-rows: 1 + :widths: 35 65 + + * - File + - Description + * - :file:`yamnet_label_list.txt` + - 521 lines; one AudioSet class name per line for YAMNet + * - :file:`vggish_label_list.txt` + - 10 lines; one UrbanSound8K class name per line for VGGish + +*********** +Input Audio +*********** + +All pipelines expect **16 kHz, mono, signed 16-bit PCM WAV** input. Sample files in +this format are pre-installed under :file:`/usr/share/tvm_inference/input/`: + +.. list-table:: + :header-rows: 1 + :widths: 35 25 40 + + * - File + - Used with + - Description + * - :file:`input_audio.wav` + - GCRN + - Sample noisy speech recording + * - :file:`speech_whistling2.wav` + - YAMNet + - Sample audio for event classification + * - :file:`urbansound_16k.wav` + - YAMNet, VGGish + - Sample urban sound recording + +If your source audio is in a different format, convert it before use: + +.. code-block:: bash + + ffmpeg -i input.wav -ar 16000 -ac 1 -sample_fmt s16 output_16k.wav + +Or convert inline inside a GStreamer pipeline by inserting ``audioconvert`` and +``audioresample`` before the ``audio/x-raw`` capsfilter. + +********************** +Pre-Packaged Libraries +********************** + +The following libraries are pre-installed on the AM62DX EdgeAI image. No installation +is required to run the out-of-box pipelines on the AM62D EVM. + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Library + - Description + * - ``libti_rpmsg_dma`` + - RPMsg and DMA client library for inter-processor communication + * - :file:`libtvm_runtime.so` + - Apache TVM ARM64 runtime for model inference + * - :file:`libgstti.so` + - GStreamer plugin library containing ``tidspkernel`` and ``titvm`` elements + * - ``libasound`` + - ALSA audio library + * - ``libsndfile`` + - Audio file I/O library + * - ``libjson-c`` + - JSON parsing library + +************ +Model Daemon +************ + +The ``tvm_model_daemon`` keeps a TVM model loaded in memory and serves inference +requests over a Unix domain socket at ``/var/run/tvm-inference.sock``. Both the Linux +inference application and the GStreamer ``titvm`` element connect to it automatically, +eliminating repeated artifact loading for each run. + +Start the daemon: + +.. code-block:: console + + root@am62dxx-evm:~# systemctl enable --now tvm-model-daemon.service + root@am62dxx-evm:~# journalctl -u tvm-model-daemon -f + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: rpmsg: Opened endpoint device /dev/rpmsg4 + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: [daemon] Loading TVM artifacts from: /usr/share/tvm_inference/artifacts/gcrn + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: [daemon] C7x firmware OK and already running + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: c7x: Connected to compute service + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: c7x: Staging buffer: 0xffff6c89e000 (phys 0xa3000000) + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: c7x: Result buffer: 0xffff7549e000 (phys 0xabc00000) + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: [08:05:48] {TVM_HOME}/src/runtime/contrib/tidl/tidl_runtime_rpmsg.cc:216: TVM AM62D: ping status=0 version=0x20000 uptime=17438ms + Oct 23 08:05:48 am62dxx-evm tvm_model_daemon[997]: [08:05:48] {TVM_HOME}/src/runtime/contrib/tidl/tidl_runtime_rpmsg.cc:239: TVM AM62D: loading deploy_mod (51527424 bytes) with 1 input name(s) + Oct 23 08:05:49 am62dxx-evm tvm_model_daemon[997]: c7x: Loaded module handle=1 (text=0 data=0 input_offset=51527443) + Oct 23 08:05:49 am62dxx-evm tvm_model_daemon[997]: [08:05:49] {TVM_HOME}/src/runtime/contrib/tidl/tidl_runtime_rpmsg.cc:263: TVM AM62D: module loaded, handle=1 + +To preload artifacts at boot: + +.. code-block:: console + + root@am62dxx-evm:~# systemctl enable --now tvm-model-preload.service + +If the daemon is not running, both the Linux application and GStreamer ``titvm`` element +will wait up to 30 seconds before falling back to loading the model in-process. + +****************** +DSP Firmware Check +****************** + +Both the Linux application and GStreamer pipelines require the C7x™ DSP firmware to be +running. Verify before use: + +.. code-block:: console + + root@am62dxx-evm:~# cat /sys/class/remoteproc/remoteproc0/state + running diff --git a/source/devices/AM62DX/edgeai/out_of_box_demo.rst b/source/devices/AM62DX/edgeai/out_of_box_demo.rst new file mode 100644 index 000000000..59120a859 --- /dev/null +++ b/source/devices/AM62DX/edgeai/out_of_box_demo.rst @@ -0,0 +1,282 @@ +.. _am62dx_edgeai_oob_demo: + +======================= +Out-of-Box Web GUI Demo +======================= + +The AM62DX EVM includes an out-of-box web-based GUI that demonstrates EdgeAI +capabilities directly from a browser. Once the board is powered on and connected to +a network, the web portal can be accessed from any device on the same network without +any additional setup. + +The portal showcases the three core EdgeAI pipelines — home screen, speech +enhancement, and audio classification — running on the C7x™ DSP. + +OOB Web GUI Home Screen +======================= + +.. figure:: /images/am62dx_web_gui_home.png + :align: center + :width: 80% + + AM62DX out-of-box web GUI home screen + +This is the Sitara AM62D DSP + Edge AI Portal, a web-based user interface for managing and +evaluating Texas Instruments' AM62D Evaluation Module. The dashboard allows developers +to launch real-time audio analytics and DSP compute demos, and monitor live system +telemetry like CPU load, memory footprints, and SDK software versions. + +To open the EdgeAI portal, follow these steps: + +1. Once the AM62D EdgeAI image is flashed (see :ref:`pub_edgeai_am62dxx_flashing`), get the IP address of the EVM using ``ip addr`` +2. Open a browser and navigate to ``:3000`` +3. This starts the web server. + +The left navigation sidebar starts with ``Home``, then groups the rest into three sections: + +- **Demos** — ``Audio Intelligence`` (speech enhancement and audio classification) and + ``DSP Acceleration`` (2D FFT, biquad filter chain, and workload offload via RPMsg-DMA + compute demos). +- **Tools** — ``AI Model Inspector`` to upload and browse TIDL (TI Deep Learning) model + inspection reports, including architecture, layer details, and performance metrics. +- **System** — ``Device Info``, ``Logs``, and ``Help``. + +The main panel opens with a welcome banner — ``SITARA AM62D DSP + Edge AI Portal`` — and a +**GET STARTED** button that jumps straight to the Audio Intelligence demo. Below it, an +**Explore Demos And Tools** panel opens or shows the same three entry points (Audio +Intelligence, DSP Acceleration, AI Model Inspector) as clickable cards with a one-line +description each. + +The home screen surfaces the board's connection state and identity at a glance: + +- **Connection status** — upper-right banner shows ``Connected to AM62D EVM`` along with the + board's IP address. +- **Info panel** — current ``SDK Version``, ``MCU+ SDK Version``, and ``TIDL Version`` + installed on the board. +- **Device Info panel** — ``Device``, ``Board``, ``SoC``, ``IP Address``, and system ``Uptime``. +- **Runtime Status panel** — live ``CPU Load``, ``CPU Avg``, ``CPU Peak``, and ``RAM + Used/Free``, refreshed continuously while the portal is open. + +A status bar at the bottom of every page repeats ``Uptime``, ``CPU Load``, and ``RAM`` for +quick reference while you are deep in a demo page, along with the portal version, build date, +and a link to the full documentation. + +A theme toggle in the top navigation bar switches the entire portal between dark mode +(default) and light mode. The selection applies immediately across all pages and persists +as you navigate between demos. + +Speech Enhancement Demo +======================= + +.. figure:: /images/am62dx_web_gui_speech_enhancement.png + :align: center + :width: 80% + + AM62DX web GUI — speech enhancement demo + +This demonstration showcases real-time speech enhancement. It integrates noise reduction +powered by the C7x™ DSP to significantly improve speech quality. + +Under ``Demo Selection`` on the left, three options are available: + +- **Speech Enhancement** — C7x™ DSP signal processing + GCRN model inference +- **Audio Classification** — C7x™ DSP + YAMNet/VGGish classification models +- **Custom Audio Pipeline** — a generic ``gst-launch`` runner with artifact management for + running custom GStreamer pipelines from the portal. + +The ``About`` and ``Model Info`` collapsible panels describe the demo and the underlying +GCRN-based noise-suppression model. The ``Audio Source`` card shows the active input file +(for example :file:`input_audio.wav` — 1ch, 16 kHz, 16-bit, 9.77s) with a ``Default`` tag, and a +status indicator (``Ready``, ``Running``, etc.). + +To run the speech enhancement demo: + +1. Click **Run Demo** in the upper-right corner to begin the process. The status line below the + Audio Source card changes from ``Ready`` to running, and the button itself becomes + **Stop Demo**. +2. Allow the system to complete the processing cycle. +3. Once finished, the interface will display both the original input and the processed output audio files. +4. Use the play button on the input file to hear the original noisy audio. +5. Use the play button on the output file to hear the enhanced version. + +**Speech Enhancement Result** + +The ``Visualization`` section plots both a spectrogram and a waveform for the ``Input +(Noisy)`` and ``Output (Enhanced)`` audio, stacked one above the other, with hover tooltips +reporting time, frequency, and spectral level (or sample amplitude for the waveform). +Spectrogram controls let you switch between ``per-plot peak`` (auto-scale each spectrogram independently) +and ``fixed -80 to 0 dB`` color scale, zoom the time axis, and **SAVE ZIP** to download the plotted data +and waveforms for offline analysis. + +.. figure:: /images/am62dx/edgeai/web_speech_enhancement.gif + :align: center + :width: 80% + + AM62DX web GUI — speech enhancement demo Result 1 (input versus enhanced spectrograms) + + +.. figure:: /images/am62dx_web_gui_speech_enhancement_result_2.png + :align: center + :width: 80% + + AM62DX web GUI — speech enhancement demo Result 2 (input versus enhanced waveforms, and + per-stage processing times) + +Below the waveforms, a ``Processing Times`` table breaks down DSP/TIDL latency per +processed frame range (for example ``0-400``, ``301-701``, ``602-1002`` — overlapping +ranges reflecting the overlap-save chunking described in +:ref:`pub_edgeai_am62dx_enhancement_chunking`) into ``Pre-Processing (STFT)``, ``Model +Inference (GCRN)``, ``Post-Processing (ISTFT)``, and ``Total Time``. In a typical run, STFT +and ISTFT each take on the order of 10-20 ms per chunk while the GCRN model inference step +dominates the total, so this table is the quickest way to see where latency budget on the +DSP is actually going. + + +Audio Classification Demo +========================= + +.. figure:: /images/am62dx_web_gui_audio_classification.png + :align: center + :width: 80% + + AM62DX web GUI — audio classification demo + +This demo demonstrates audio classification using YAMNet and VGGish models on the AM62D C7x™ DSP. + +The ``Model`` panel lets you pick the ``Classification Model`` — ``YAMNet`` (521 AudioSet +classes, MobileNet v1) or ``VGGish`` (10 UrbanSound8K classes, VGGish features); the panel +shows this class-count and backbone caption under whichever model is selected, so you can +confirm the right one before running. Unlike Speech Enhancement, which defaults to a sample +WAV file, Audio Classification defaults to live input: the ``Audio Source`` card shows +``AM62D2-EVM: pcmdevice-codec-0`` tagged ``Live Microphone`` (live ALSA capture, 16 kHz mono +PCM, 1s chunks). The ``Input Source`` dropdown lets you switch to a different captured device +with ``REFRESH``, or classify a recorded clip instead with ``UPLOAD WAV FILE``. + +To run the audio classification demo: + +1. Select Audio Classification from the Demo Selection menu on the left. +2. Select Model from the model list — YAMNet or VGGish. +3. Select the audio source: leave it on the live microphone, or pick/upload a WAV file. +4. Click **Run Demo** in the upper-right corner; the status line shows ``Idle`` before you start. +5. Check the audio classification result in the output section below. + +While running, a live status line (``Running Audio Classification``) replaces ``Idle`` above +the results, and the upper-right button becomes **Stop Demo**. The ``Classification Results`` +panel lists each detected event with a running count and a wall-clock timestamp, newest first +(for example ``13 Silence 02:11:06 pm``), and can be cleared with the trash icon. Below it, a +``Top-10 Predictions Over Patches`` heatmap plots the model's top classes against successive +audio patches over time, with a logit-value color scale for confidence — useful for seeing +which classes the model was close to choosing, not just the winning one. + +**YAMNET Model Result** + +.. figure:: /images/am62dx/edgeai/web_audio_classification_yamnet.gif + :align: center + :width: 80% + + AM62DX web GUI — audio classification YAMNet demo. + +**VGGish Model Result** + +.. figure:: /images/am62dx/edgeai/web_audio_classification_vggish.gif + :align: center + :width: 80% + + AM62DX web GUI — audio classification VGGish demo + +Custom Audio GStreamer Pipeline +=============================== + +The ``Custom Audio Pipeline`` demo in the sidebar lets you run arbitrary GStreamer +pipelines directly on the AM62D. This is useful for prototyping custom audio processing +chains, debugging GCRN-based models with custom parameters, or testing YAMNet/VGGish +classification on different audio sources. + +To use your own trained model, see :ref:`Integrating a New Model `. + +The interface has three main areas: + +- **Pipeline Presets** — quick-start tabs that load pre-built pipelines (``Speech + Enhancement``, ``Classification (YAMNet)``, ``Classification (VGGish)``, ``Custom``). + Pre-configured pipelines are hidden behind the phrase "No saved pipelines yet — write a + command and click Save As". +- **Pipeline Command** — a full ``gst-launch-1.0`` command editor. Copy and paste buttons + let you transfer the command to the clipboard or load a saved preset. +- **Input Files** and **TVM Artifacts** — collapsible panels for staging input audio files + and model files (TIDL artifacts). + +Below the ``Pipeline Command``, a collapsible ``Input Source`` dropdown shows the device +or file to be piped into the pipeline, and a status indicator shows whether the system is +ready, running, or idle. + +Speech Enhancement Pipeline with GCRN +------------------------------------- + +This preset runs a GCRN-based noise-suppression pipeline using the C7x™ DSP and TIDL runtime. + +**To run the GCRN speech enhancement pipeline:** + +1. Click the ``Speech Enhancement`` preset tab to load the pre-built command. +2. The ``Pipeline Command`` box displays the full GStreamer command. +3. Click ``Run Demo`` to start the pipeline. +4. The ``Pipeline Output`` panel displays processing logs, including DSP execution times, + model inference latency, and phase transitions (``PREROLLED``, ``PLAYING``, etc.). +5. After execution, the output file is ready for playback — download it or use the play + button to verify the noise reduction. + +.. figure:: /images/am62dx/edgeai/web_gst_speech_enhancement_gcrn.gif + :align: center + :width: 80% + + Output from the GCRN pipeline + +Audio Classification with YAMNet +-------------------------------- + +This preset runs YAMNet (AudioSet, 521 classes) using the C7x™ DSP for real-time audio +event classification. + +**To run the YAMNet classification pipeline:** + +1. Click the ``Classification (YAMNet)`` tab. +2. The ``Pipeline Command`` box displays the full GStreamer command for YAMNet classification. +3. Choose the audio source — a file or live microphone — from the ``Input Source`` dropdown. +4. Click ``Run Demo``. +5. The ``Classification Results`` output shows top-3 predictions per audio window with + confidence logits; execution ends after processing the full input. + +.. figure:: /images/am62dx_web_gui_speech_classification_gst_yamnet.png + :align: center + :width: 80% + + YAMNet pipeline command + +.. figure:: /images/am62dx/edgeai/web_gst_audio_classification_yamnet.gif + :align: center + :width: 80% + + YAMNet inference output showing top-3 predictions + +Audio Classification with VGGish +-------------------------------- + +This preset runs VGGish (UrbanSound8K, 10 classes) for environmental sound classification. + +**To run the VGGish classification pipeline:** + +1. Click the ``Classification (VGGish)`` tab. +2. The command is similar to YAMNet but uses VGGish model weights and UrbanSound8K class labels. +3. Select your audio source and click ``Run Demo``. +4. The output lists detected events with confidence scores for each of the 10 UrbanSound8K classes. + +.. figure:: /images/am62dx_web_gui_speech_classification_gst_vggish.png + :align: center + :width: 80% + + VGGish classification pipeline command + +.. figure:: /images/am62dx/edgeai/web_gst_audio_classification_vggish.gif + :align: center + :width: 80% + + VGGish inference output diff --git a/source/devices/AM62DX/edgeai/overview.rst b/source/devices/AM62DX/edgeai/overview.rst new file mode 100644 index 000000000..90cd4bda3 --- /dev/null +++ b/source/devices/AM62DX/edgeai/overview.rst @@ -0,0 +1,54 @@ +.. _am62dx_edgeai_overview: + +######## +Overview +######## + +**Welcome to Processor SDK Linux Edge AI for AM62DX!** + +The SDK provides software and tools to run high-performance deep learning inference +on Texas Instruments' AM62DX processor. It enables a practical Edge AI solution by +combining a high-performance A53 ARM processor with a dedicated DSP optimized for +neural network inference and digital signal processing. This Edge AI inference chain +is a common pattern across TI devices, allowing efficient on-device AI workloads with +real-time performance. The AM62DX specifically supports audio modality to run +audio analytics pipelines such as speech enhancement and audio classification +on the DSP. + +.. figure:: /images/am62dx/edgeai/web_home_screen.gif + :scale: 30 + :align: center + + AM62DX out-of-box web GUI home screen + +.. figure:: /images/edgeai/edgeai-sdk-feature.png + :scale: 50 + :align: center + + Processor SDK Linux Edge AI for AM62DX feature overview + +The SDK enables an interplay of open-source components such as GStreamer and +Apache TVM. The SDK supports editing, building, and debugging applications directly +on the target, as well as on a host PC where applications are cross-compiled and +built before being deployed to the target. + +.. figure:: /images/am62dx_open_src_comp.png + :scale: 50 + :align: center + + Industry Standard Components supported in Processor SDK Linux Edge AI for AM62DX + +The SDK mainly comprises of three parts as shown in the illustration below. + +- **Edge AI application stack** — Linux-side applications and GStreamer pipelines + that drive audio inference using TVM-RT and open-source components. +- **Foundational Linux components** — AArch64 Linux , secure boot, filesystem, and Linux + drivers that form the base platform. +- **MCU+ SDK** — firmware framework for the C7x™ DSP, handles signal chains (STFT/ISTFT, pre/post-processing) + and model dispatch. Includes build system and flashing utilities. + +.. figure:: /images/am62dx_sdk_overview.png + :scale: 80 + :align: center + +To get started with the setup click the **Next** button. diff --git a/source/devices/AM62DX/edgeai/performance.rst b/source/devices/AM62DX/edgeai/performance.rst new file mode 100644 index 000000000..242f56a04 --- /dev/null +++ b/source/devices/AM62DX/edgeai/performance.rst @@ -0,0 +1,136 @@ +.. include:: + +.. _edgeai_performance: + +##################### +Performance Benchmark +##################### + +This section reports **inference performance** - latency and memory - for the +reference |__PART_FAMILY_NAME__| EdgeAI models. The same methodology (perfsim +estimate + on-target measured) applies to any model you compile through the +workflow. It is **not** an accuracy or audio-quality benchmark: model quality +(classification accuracy, PESQ / STOI / SI-SDR) is measured by the ``evaluate`` +step in :ref:`Model Compile and Evaluate `. + +*********** +Methodology +*********** + +Two independent figures are reported for each model, and they must not be +confused: + +.. list-table:: + :header-rows: 1 + :widths: 22 78 + + * - Figure + - How it is obtained + * - **perfsim estimate** + - Produced by the TIDL performance simulator during ``compile``. The + compiler writes a :file:`result.yaml` per model under its work directory; + the ``perfsim`` latency, GMAC count, and estimated DDR transfer are + read directly from it. This is a *static estimate* for the C7\ |trade| + NPU subgraph - it does not include Arm-side pre/post-processing or + runtime overhead. + * - **on-target measured** + - Wall-clock latency timed on the AM62D target while running the + :ref:`Model Zoo ` inference scripts. For the + streaming speech-enhancement models it is also expressed as a + **real-time factor (RTF)** = processing time / audio duration; RTF < 1 + means the model keeps up with real-time audio. + +Only models compiled for the C7\ |trade| NPU (TVM-RT + TIDL) have a perfsim +estimate. GTCRN runs FP32 on the Arm core through the ONNX runtime and has no +TIDL subgraph, so perfsim does not apply to it. + +.. note:: + + The perfsim values below are read from :file:`result.yaml` and are labeled + *estimated (perfsim)*. On-target measured latency and RTF are marked **TBD** + until measured on the AM62D EVM. + +***************** +Per-model results +***************** + +.. list-table:: + :header-rows: 1 + :widths: 16 12 12 20 14 26 + + * - Model + - Model ID + - Precision + - perfsim latency *(estimated)* + - GMACs + - DDR transfer *(estimated)* + * - VGGish11 + - ``ac-0101`` + - 8-bit + - 5.235 ms + - 1.1515 + - 10.73 MB + * - YAMNet + - ``ac-0201`` + - 8-bit + - 1.990 ms + - 0.0692 + - 3.55 MB + * - GCRN + - ``ase-0201`` + - 16-bit + - TBD + - TBD + - TBD + * - GTCRN + - ``ase-0100`` + - FP32 (Arm) + - N/A + - N/A + - N/A + +VGGish11 and YAMNet are compiled and have :file:`result.yaml` perfsim values (shown +above). GCRN is compiled at 16-bit but its :file:`result.yaml` is not yet captured, +so its perfsim figures are **TBD**. GTCRN is Arm-only (no TIDL subgraph), so +perfsim is **N/A**. + +On-target measured latency and RTF for all four models are **TBD** - to be +filled in from the Model Zoo inference scripts on the AM62D EVM: + +.. list-table:: + :header-rows: 1 + :widths: 20 20 30 30 + + * - Model + - On target + - Measured latency + - RTF + * - VGGish11 + - C7\ |trade| NPU + - TBD + - - (clip classifier) + * - YAMNet + - C7\ |trade| NPU + - TBD + - - (clip classifier) + * - GCRN + - C7\ |trade| NPU + - TBD + - TBD + * - GTCRN + - Arm core + - TBD + - TBD + +RTF is reported only for the streaming speech-enhancement models (GCRN, GTCRN); +VGGish11 and YAMNet classify a whole clip, so a real-time factor is not +meaningful for them. + +******************* +Scope & Limitations +******************* + +This section does **not** assess model quality. Classification accuracy and +speech-enhancement quality metrics are produced by ``tidlrunner-cli evaluate`` +and documented in +:ref:`Model Compile and Evaluate `. diff --git a/source/devices/AM62DX/edgeai/version.txt b/source/devices/AM62DX/edgeai/version.txt new file mode 100644 index 000000000..9df24b162 --- /dev/null +++ b/source/devices/AM62DX/edgeai/version.txt @@ -0,0 +1 @@ +12_01_00_00 diff --git a/source/images/am62dx/edgeai/pipeline_gcrn.png b/source/images/am62dx/edgeai/pipeline_gcrn.png new file mode 100644 index 000000000..7e1132888 Binary files /dev/null and b/source/images/am62dx/edgeai/pipeline_gcrn.png differ diff --git a/source/images/am62dx/edgeai/pipeline_vggish.png b/source/images/am62dx/edgeai/pipeline_vggish.png new file mode 100644 index 000000000..7d9fb9cf4 Binary files /dev/null and b/source/images/am62dx/edgeai/pipeline_vggish.png differ diff --git a/source/images/am62dx/edgeai/pipeline_yamnet.png b/source/images/am62dx/edgeai/pipeline_yamnet.png new file mode 100644 index 000000000..501ef01fd Binary files /dev/null and b/source/images/am62dx/edgeai/pipeline_yamnet.png differ diff --git a/source/images/am62dx/edgeai/web_audio_classification_vggish.gif b/source/images/am62dx/edgeai/web_audio_classification_vggish.gif new file mode 100644 index 000000000..1bca8758c Binary files /dev/null and b/source/images/am62dx/edgeai/web_audio_classification_vggish.gif differ diff --git a/source/images/am62dx/edgeai/web_audio_classification_yamnet.gif b/source/images/am62dx/edgeai/web_audio_classification_yamnet.gif new file mode 100644 index 000000000..db82e6dcb Binary files /dev/null and b/source/images/am62dx/edgeai/web_audio_classification_yamnet.gif differ diff --git a/source/images/am62dx/edgeai/web_gst_audio_classification_vggish.gif b/source/images/am62dx/edgeai/web_gst_audio_classification_vggish.gif new file mode 100644 index 000000000..591d273ca Binary files /dev/null and b/source/images/am62dx/edgeai/web_gst_audio_classification_vggish.gif differ diff --git a/source/images/am62dx/edgeai/web_gst_audio_classification_yamnet.gif b/source/images/am62dx/edgeai/web_gst_audio_classification_yamnet.gif new file mode 100644 index 000000000..f3fdc661f Binary files /dev/null and b/source/images/am62dx/edgeai/web_gst_audio_classification_yamnet.gif differ diff --git a/source/images/am62dx/edgeai/web_gst_speech_enhancement_gcrn.gif b/source/images/am62dx/edgeai/web_gst_speech_enhancement_gcrn.gif new file mode 100644 index 000000000..638ec2a37 Binary files /dev/null and b/source/images/am62dx/edgeai/web_gst_speech_enhancement_gcrn.gif differ diff --git a/source/images/am62dx/edgeai/web_home_screen.gif b/source/images/am62dx/edgeai/web_home_screen.gif new file mode 100644 index 000000000..7d5d8c1d8 Binary files /dev/null and b/source/images/am62dx/edgeai/web_home_screen.gif differ diff --git a/source/images/am62dx/edgeai/web_speech_enhancement.gif b/source/images/am62dx/edgeai/web_speech_enhancement.gif new file mode 100644 index 000000000..121427d6a Binary files /dev/null and b/source/images/am62dx/edgeai/web_speech_enhancement.gif differ diff --git a/source/images/am62dx_board_config.png b/source/images/am62dx_board_config.png new file mode 100644 index 000000000..e9504d1dd Binary files /dev/null and b/source/images/am62dx_board_config.png differ diff --git a/source/images/am62dx_boot_pin.png b/source/images/am62dx_boot_pin.png new file mode 100644 index 000000000..d4a2bad9c Binary files /dev/null and b/source/images/am62dx_boot_pin.png differ diff --git a/source/images/am62dx_edgeai.png b/source/images/am62dx_edgeai.png new file mode 100644 index 000000000..c56cc4f3f Binary files /dev/null and b/source/images/am62dx_edgeai.png differ diff --git a/source/images/am62dx_open_src_comp.png b/source/images/am62dx_open_src_comp.png new file mode 100644 index 000000000..85d296304 Binary files /dev/null and b/source/images/am62dx_open_src_comp.png differ diff --git a/source/images/am62dx_sdk_overview.png b/source/images/am62dx_sdk_overview.png new file mode 100644 index 000000000..826b6cd5f Binary files /dev/null and b/source/images/am62dx_sdk_overview.png differ diff --git a/source/images/am62dx_web_gui_audio_classification.png b/source/images/am62dx_web_gui_audio_classification.png new file mode 100644 index 000000000..239067bef Binary files /dev/null and b/source/images/am62dx_web_gui_audio_classification.png differ diff --git a/source/images/am62dx_web_gui_audio_classification_vggish_result.png b/source/images/am62dx_web_gui_audio_classification_vggish_result.png new file mode 100644 index 000000000..868400a42 Binary files /dev/null and b/source/images/am62dx_web_gui_audio_classification_vggish_result.png differ diff --git a/source/images/am62dx_web_gui_audio_classification_yamnet_result.png b/source/images/am62dx_web_gui_audio_classification_yamnet_result.png new file mode 100644 index 000000000..f04772509 Binary files /dev/null and b/source/images/am62dx_web_gui_audio_classification_yamnet_result.png differ diff --git a/source/images/am62dx_web_gui_home.png b/source/images/am62dx_web_gui_home.png new file mode 100644 index 000000000..e1d66e205 Binary files /dev/null and b/source/images/am62dx_web_gui_home.png differ diff --git a/source/images/am62dx_web_gui_speech_classification_gst_vggish.png b/source/images/am62dx_web_gui_speech_classification_gst_vggish.png new file mode 100644 index 000000000..4fd74f74c Binary files /dev/null and b/source/images/am62dx_web_gui_speech_classification_gst_vggish.png differ diff --git a/source/images/am62dx_web_gui_speech_classification_gst_yamnet.png b/source/images/am62dx_web_gui_speech_classification_gst_yamnet.png new file mode 100644 index 000000000..c6fdf69e4 Binary files /dev/null and b/source/images/am62dx_web_gui_speech_classification_gst_yamnet.png differ diff --git a/source/images/am62dx_web_gui_speech_enhancement.png b/source/images/am62dx_web_gui_speech_enhancement.png new file mode 100644 index 000000000..89b0a774e Binary files /dev/null and b/source/images/am62dx_web_gui_speech_enhancement.png differ diff --git a/source/images/am62dx_web_gui_speech_enhancement_result_1.png b/source/images/am62dx_web_gui_speech_enhancement_result_1.png new file mode 100644 index 000000000..c5a0006b7 Binary files /dev/null and b/source/images/am62dx_web_gui_speech_enhancement_result_1.png differ diff --git a/source/images/am62dx_web_gui_speech_enhancement_result_2.png b/source/images/am62dx_web_gui_speech_enhancement_result_2.png new file mode 100644 index 000000000..ee1d57b9d Binary files /dev/null and b/source/images/am62dx_web_gui_speech_enhancement_result_2.png differ diff --git a/source/images/am62dx_webgui_2dfft.png b/source/images/am62dx_webgui_2dfft.png new file mode 100644 index 000000000..19dae095e Binary files /dev/null and b/source/images/am62dx_webgui_2dfft.png differ diff --git a/source/images/am62dx_webgui_cascade_biquad.png b/source/images/am62dx_webgui_cascade_biquad.png new file mode 100644 index 000000000..1d718df21 Binary files /dev/null and b/source/images/am62dx_webgui_cascade_biquad.png differ diff --git a/source/images/am62dx_webgui_dspoffload.png b/source/images/am62dx_webgui_dspoffload.png new file mode 100644 index 000000000..5a25f5975 Binary files /dev/null and b/source/images/am62dx_webgui_dspoffload.png differ diff --git a/source/images/audioai/tvm_tidl_flow.svg b/source/images/audioai/tvm_tidl_flow.svg new file mode 100644 index 000000000..7d5920868 --- /dev/null +++ b/source/images/audioai/tvm_tidl_flow.svg @@ -0,0 +1,4 @@ + + + +
ONNX audio model
Model import into TVM
Partitioning
TIDL import API
TVM C7™ DSP
code generation
TIDL network compiler
TVM-RT deployable artifact
(deploy_lib.so + graph.json + params)
compile time (x86)
run time (AM62D)
TVM runtime
TIDL-RT
C7™ NPU
TIDL unsupported ops
TIDL supported ops
Text is not SVG - cannot display \ No newline at end of file diff --git a/source/images/edgeai/edgeai-sdk-feature.png b/source/images/edgeai/edgeai-sdk-feature.png index 9a2a45ab7..19b6262c3 100644 Binary files a/source/images/edgeai/edgeai-sdk-feature.png and b/source/images/edgeai/edgeai-sdk-feature.png differ diff --git a/source/linux/Demo_User_Guides/AM62D_2DFFT_User_Guide.rst b/source/linux/Demo_User_Guides/AM62D_2DFFT_User_Guide.rst index b9735b12d..c7eab507d 100644 --- a/source/linux/Demo_User_Guides/AM62D_2DFFT_User_Guide.rst +++ b/source/linux/Demo_User_Guides/AM62D_2DFFT_User_Guide.rst @@ -60,7 +60,7 @@ Below figure shows how this demo works: processed output data is available in the shared DDR buffer. - Step 7: A53 reads back processed data from DMA buffer - - A53 copies the processed data from the shared buffer for validation. + - A53 copies the processed data from the shared buffer for validation. - Step 8: Validation and Performance Reporting - The application compares the output data against expected results to verify correctness. @@ -79,12 +79,12 @@ Hardware prerequisites - USB to Universal asynchronous receiver-transmitter (UART) cable for console access - PC (Windows or Linux) to flash image onto an SD Card - Host PC Requirements: - + - Operating system: - + - Windows: |__WINDOWS_SUPPORTED_LONG__| - Ubuntu: |__LINUX_UBUNTU_VERSION_LONG__| - + - Memory: Minimum 4GB RAM (8GB or more recommended) - Storage: At least 10GB of free space @@ -105,10 +105,10 @@ EVM setup ********* #. Cable Connections - + - The figure below shows some important cable connections, ports and switches. - Take note of the location of the "BOOTMODE" switch for SD card boot mode. - + .. figure:: /images/AM62D_evm_setup.png :height: 600 :width: 1000 @@ -116,14 +116,14 @@ EVM setup - Refer `Quick Start Guide for AM62D2-EVM `__ for more details. #. Setup UART Terminal - + - First identify the UART port as enumerated on the host machine. - Connect the EVM and UART cable to the UART to USB port as shown in cable connections. - In Windows, you can use the "Device Manager" to see the detected UART ports: - + - Search "Device Manager" in Windows Search Box in the Windows taskbar. - + - If you do not see any USB serial ports listed in "Device Manager" under "Ports (COM & LPT)", then make sure you have installed the UART to USB driver from `FTDI `__. @@ -131,9 +131,9 @@ EVM setup keep other options to default and set 115200 baud rate. #. Setup SD card Boot Mode - + - EVM SD card boot mode setting: - + - BOOTMODE [ 8 : 15 ] (SW3) = 0100 0000 - BOOTMODE [ 0 : 7 ] (SW2) = 1100 0010 @@ -141,11 +141,47 @@ EVM setup Steps to validate 2D FFT DSP offload demo ***************************************** +There are two ways to run and visualize this demo: + +Using Web GUI +============= + +The AM62D EVM includes a web-based GUI portal that provides an out-of-box way to +run and visualize the 2D FFT demo from any browser on the same network — no host +utility installation required. + +#. Flash and boot the EVM as described in the EVM setup section above. + +#. Get the EVM IP address: + + .. code-block:: console + + root@am62dxx-evm:~# ifconfig + +#. Open a browser on your host PC and navigate to ``http://:3000`` + +#. On the home screen, click **DSP Acceleration** or select it from the left sidebar + under **Demos**. + +#. Select **2D FFT** from the demo options and click the run button to start the demo. + +#. The portal displays the test result (PASSED/FAILED) along with performance metrics: + DSP load, cycle count, and DDR throughput. + +.. figure:: /images/am62dx_webgui_2dfft.png + :align: center + :width: 80% + + AM62D Web GUI — 2D FFT demo + +Using Console +============= + #. Flash an SD card with the :file:`tisdk-default-image-rt-am62dxx-evm.rootfs.wic.xz` image and follow the instructions provided at :ref:`Create SD Card ` guide. -#. Insert the flashed SD card into the `AUDIO-AM62D-EVM `__ - and power on the TI AUDIO-AM62D-EVM. +#. Insert the flashed SD card into the `AM62D EVM `__ + and power on the TI AM62D EVM. #. Make sure the EVM boot mode switches are positioned for SD card boot as described earlier. @@ -158,30 +194,30 @@ Steps to validate 2D FFT DSP offload demo Just enter "root" to log in. #. Run the 2D FFT DSP offload demo application from the console: - + .. code-block:: console - + root@am62dxx-evm:~# rpmsg_2dfft_example #. The application will execute and display the results: - + .. code-block:: console - + RPMsg based 2D FFT Offload Example - + ***************************************** ***************************************** - + C7x 2DFFT Test PASSED C7x Load: 1% C7x Cycle Count: 327000 C7x DDR Throughput: 0.801656 MB/s - + ***************************************** ***************************************** .. note:: - + The test reports "PASSED" if the computed 2D FFT output matches the expected results within tolerance (0.01), otherwise it reports "FAILED". @@ -209,44 +245,44 @@ Building the linux demo binary from sources #. The source code for the 2D FFT DSP offload demo is available as part of the `rpmsg-dma `__. - + .. code-block:: console - + host# git clone https://github.com/TexasInstruments/rpmsg-dma.git -b main #. Download and Install the AM62D Linux SDK from |__SDK_DOWNLOAD_URL__| following the steps mentioned at :ref:`Download and Install the SDK `. #. Prepare the environment for cross compilation. - + .. code-block:: console - + host# source /linux-devkit/environment-setup #. Compile the source: - + .. code-block:: console - + [linux-devkit]:> cd [linux-devkit]:> cmake -S . -B build; cmake --build build - + - This command builds: - + - The example application :file:`rpmsg_2dfft_example` - + - Transfer the generated files to SD card: - + - The example binary :file:`rpmsg_2dfft_example` to :file:`/usr/bin` - The test input data file :file:`2dfft_input_data.bin` to :file:`/usr/share/2dfft_test_data/` - The expected output data file :file:`2dfft_expected_output_data.bin` to :file:`/usr/share/2dfft_test_data/` - The C7 DSP firmware file :file:`fft2d_linux_dsp_offload_example.c75ss0-0.release.strip.out` to :file:`/lib/firmware/` - + - Optional: - + - To build only the library or only the example, use: - + .. code-block:: console - + cmake -S . -B build -DBUILD_LIB=OFF # disables library build cmake -S . -B build -DBUILD_AUDIO_OFFLOAD_EXAMPLE=OFF # disables audio_offload example build cmake -S . -B build -DBUILD_2DFFT_OFFLOAD_EXAMPLE=OFF # disables 2dfft_offload example build diff --git a/source/linux/Demo_User_Guides/AM62D_Cascade_Biquad_Parametric_EQ_User_Guide.rst b/source/linux/Demo_User_Guides/AM62D_Cascade_Biquad_Parametric_EQ_User_Guide.rst index 08ee5278b..d2d79d887 100644 --- a/source/linux/Demo_User_Guides/AM62D_Cascade_Biquad_Parametric_EQ_User_Guide.rst +++ b/source/linux/Demo_User_Guides/AM62D_Cascade_Biquad_Parametric_EQ_User_Guide.rst @@ -161,9 +161,46 @@ EVM Setup Steps to run Cascade Biquad Parametric EQ demo ********************************************** +There are two ways to run and visualize this demo: + +Using Web GUI +============= + +The AM62D EVM includes a web-based GUI portal that provides an out-of-box way to +run and visualize the Cascade Biquad Parametric EQ demo from any browser on the same +network — no host utility installation required. + +#. Flash and boot the EVM as described in the EVM Setup section above. + +#. Get the EVM IP address: + + .. code-block:: console + + root@am62dxx-evm:~# ifconfig + +#. Open a browser on your host PC and navigate to ``http://:3000`` + +#. On the home screen, click **DSP Acceleration** or select it from the left sidebar + under **Demos**. + +#. Select **Cascade Biquad** from the demo options and click the run button to start + the demo. + +#. The portal displays real-time metrics including DSP load, cycle count, throughput, + and EQ filter controls. + +.. figure:: /images/am62dx_webgui_cascade_biquad.png + :align: center + :width: 80% + + AM62D Web GUI — Cascade Biquad Parametric EQ demo + +Using Python Host Utility +========================= + #. Flash an SD card with the :file:`tisdk-default-image-rt-am62dxx-evm.rootfs.wic.xz` image and follow the instructions provided at :ref:`Create SD Card ` guide. -#. Insert the flashed SD card to `AUDIO-AM62D-EVM `__, connect the audio input/output, Ethernet Cable and power on TI AUDIO-AM62D-EVM. +#. Insert the flashed SD card to `AM62D EVM `__, connect the audio input/output, Ethernet Cable and power on TI AM62D EVM. #. Make sure the EVM boot mode switches position set for SD card boot as described earlier diff --git a/source/linux/Demo_User_Guides/AM62D_Dsp_Offload_User_Guide.rst b/source/linux/Demo_User_Guides/AM62D_Dsp_Offload_User_Guide.rst index b48a25ec5..782d75dc4 100644 --- a/source/linux/Demo_User_Guides/AM62D_Dsp_Offload_User_Guide.rst +++ b/source/linux/Demo_User_Guides/AM62D_Dsp_Offload_User_Guide.rst @@ -35,15 +35,15 @@ Below figure shows how this demo works: .. figure:: /images/AM62D_dsp_signal_chain_audio_offload.png :height: 80 :width: 1000 - + - CasadeBiquad Low-Pass Filter: This filter is a three-stage, direct form 1 design with a low-pass cut-off frequency of 10KHz. - + - CasadeBiquad High-Pass Filter: This filter is also a three-stage, direct form 1 design with a high-pass cut-off frequency of 2KHz. - Finite Impulse Response(FIR) Low-Pass Filter: This filter is a 64-tap low-pass filter with a cut-off frequency of 8KHz. - + - Real Fast Fourier Transform(FFT) and Inverse Fast Fourier Transform(IFFT) Real: Perform FFT and IFFT of a real signal. - + - Matrix Transpose: Convert the data format between de-interleaved and interleaved formats within the signal chain. - Step 6: Processed data copied back to shared DMA Buffer (DDR) @@ -90,9 +90,9 @@ Hardware Prerequisites - Windows: |__WINDOWS_SUPPORTED_LONG__| - Ubuntu: |__LINUX_UBUNTU_VERSION_LONG__| - + - Memory: Minimum 4GB RAM (8GB or more recommended) - + - Storage: At least 10GB of free space ****************** @@ -137,7 +137,7 @@ EVM Setup - In windows, you can use the "Device Manager" to see the detected UART ports - Search "Device Manager" in Windows Search Box in the Windows taskbar. - + - If you do not see any USB serial ports listed in "Device Manager" under "Ports (COM & LPT)", then make sure you have installed the UART to USB driver from `FTDI `__. - For A53 Linux console select UART boot port (ex: COM34 in below screenshot), keep other options to default and set 115200 baud rate. @@ -147,16 +147,52 @@ EVM Setup - EVM SD card boot mode setting: - BOOTMODE [ 8 : 15 ] (SW3) = 0100 0000 - + - BOOTMODE [ 0 : 7 ] (SW2) = 1100 0010 **************************************** Steps to validate audio DSP offload demo **************************************** +There are two ways to run and visualize this demo: + +Using Web GUI +============= + +The AM62D EVM includes a web-based GUI portal that provides an out-of-box way to +run and visualize the DSP offload demo from any browser on the same network — no host +utility installation required. + +#. Flash and boot the EVM as described in the EVM Setup section above. + +#. Get the EVM IP address: + + .. code-block:: console + + root@am62dxx-evm:~# ifconfig + +#. Open a browser on your host PC and navigate to ``http://:3000`` + +#. On the home screen, click **DSP Acceleration** or select it from the left sidebar + under **Demos**. + +#. Select **DSP Offload** from the demo options and click the run button to start the demo. + +#. The portal displays real-time metrics including DSP load, filter status, input/output + spectrum and latency. + +.. figure:: /images/am62dx_webgui_dspoffload.png + :align: center + :width: 80% + + AM62D Web GUI — DSP Offload demo + +Using Python Host Utility +========================= + #. Flash an SD card with the :file:`tisdk-default-image-rt-am62dxx-evm.rootfs.wic.xz` image and follow the instructions provided at :ref:`Create SD Card ` guide. -#. Insert the flashed SD card to `AUDIO-AM62D-EVM `__, connect the 3.5mm jack headset/Speaker, Ethernet Cable and power on TI AUDIO-AM62D-EVM. +#. Insert the flashed SD card to `AM62D EVM `__, connect the 3.5mm jack headset/Speaker, Ethernet Cable and power on TI AM62D EVM. #. Make sure the EVM boot mode switches position set for SD card boot as described earlier @@ -183,14 +219,14 @@ Steps to validate audio DSP offload demo root@am62dxx-evm:~# rpmsg_audio_offload_example #. On host machine launch the :file:`audiomon.py` utility either in IP mode or UART mode - + - IP Mode - + .. code-block:: console # python audmon.py ip Ex: # python audmon.py ip 192.168.0.101 - + - UART mode .. code-block:: console @@ -203,11 +239,11 @@ Steps to validate audio DSP offload demo - Real-time visualization of: - Frame-level average amplitude (dBFS) - + - latency tracking - + - Avg load - + - Input/output FFT spectrum (only in IP mode) - Command interface to toggle features (e.g., enabling/disabling FFT filter) @@ -219,9 +255,9 @@ Steps to validate audio DSP offload demo - For more information refer: `README `_. .. note:: - + Input/output audio spectrum plotting is only supported in IP mode. UART mode supports only metrics and command interface, not audio data streams. - + Below is sample snapshot: .. figure:: /images/AM62D_host_utility_snapshot.png diff --git a/source/linux/Demo_User_Guides/AM62D_EdgeAI_User_Guide.rst b/source/linux/Demo_User_Guides/AM62D_EdgeAI_User_Guide.rst new file mode 100644 index 000000000..2a8d84ec8 --- /dev/null +++ b/source/linux/Demo_User_Guides/AM62D_EdgeAI_User_Guide.rst @@ -0,0 +1,337 @@ +.. _AM62D-edgeai-from-linux-user-guide: + +################## +AM62D Edge AI Demo +################## + +******** +Overview +******** + +This guide describes how to set up, build, and run the Edge AI inference demo on the +Texas Instruments AM62D audio evaluation module (EVM). This demo showcases real-time +audio deep learning inference on the C7x DSP, including speech enhancement and audio +classification pipelines using TVM-compiled models. The following pre-compiled models +are included out of the box: GCRN (speech enhancement), YAMNet (521-class audio event +classification), and VGGish (10-class audio classification). +Below figure shows how this demo works: + +.. figure:: /images/am62dx_edgeai.png + :height: 400 + :width: 1200 + +- Step 1: Copy sample audio data to shared DMA Buffer (DDR) + - The Linux application copies input audio data to a shared DMA buffer located + in DDR memory for communication with the C7x DSP. + +- Step 2: Notify DSP using RPMsg (IPC) + - A53 sends a control message via RPMsg (Remote Processor Messaging) to the C7x + core. This message informs the C7x that new input audio data is available for + processing in the shared DDR buffer. + +- Step 3: C7x receives RPMsg and processes the request + - The C7x DSP receives the RPMsg message, reads the buffer address and parameters, + and prepares for inference processing. + +- Step 4: C7x reads from shared DMA Buffer into L2 SRAM + - The C7x DSP copies the audio data from the DMA buffer (DDR) into its local L2 + SRAM for processing. This operation minimizes access latency compared to reading + directly from DDR. + +- Step 5: Inference execution on DSP + - Below figure shows the inference signal chain executed on C7x: + + - STFT: Converts time-domain audio frames to frequency-domain spectral frames. + - TVM Inference: Runs the selected TVM-compiled model (GCRN, YAMNet, or VGGish) + on the C7x NPU. + - ISTFT: For speech enhancement models, converts enhanced spectral output back to + time-domain audio. + +- Step 6: Inference results copied back to shared DMA Buffer (DDR) + - Once DSP processing is complete, the output (inference results or enhanced audio) + is copied back into the shared DMA buffer. + +- Step 7: C7x notifies A53 using RPMsg + - C7x sends a control message via RPMsg to the A53 core. This message informs the + A53 that processed output data is available in the shared DDR buffer. + +- Step 8: A53 receives RPMsg + - The A53 Linux application receives the RPMsg notification from the C7x confirming + that inference results are ready in the shared DDR buffer. + +- Step 9: A53 reads back results from DMA buffer + - A53 copies the inference results from the shared buffer. Depending on the pipeline: + + - **Speech enhancement**: Enhanced audio is saved to a WAV file or played back + through the TAD5212 DAC via McASP. + - **Audio classification**: Top-k class names and confidence scores are printed to + the console or displayed in the web GUI. + +********************** +Hardware Prerequisites +********************** + +- `AM62D EVM `__ + +- Audio input source (e.g., microphone) connected to J1/J2 (PCM6240 ADC) + +- Audio output device (e.g., headphones/speakers) connected to J3/J4 (TAD5212 DAC) + +- SD card (minimum 16GB) + +- USB Type-C 20W power supply (make sure to use type-C to type-C cable) + +- USB-to-UART cable for console access + +- PC (Windows or Linux) to flash image onto an SD Card + +- Ethernet cable and the expansion board `DP83867-EVM-AM `__ for network access + +- Host PC Requirements: + + - Operating system: + + - Windows: |__WINDOWS_SUPPORTED_LONG__| + + - Ubuntu: |__LINUX_UBUNTU_VERSION_LONG__| + + - Memory: Minimum 4GB RAM (8GB or more recommended) + + - Storage: At least 10GB of free space + +****************** +Software and Tools +****************** + +- TI Processor SDK Linux EdgeAI (AM62Dx) + +- MCU+ SDK for AM62Dx + +- `TI Clang Compiler Toolchain `__ + +- CMake, GCC, make, git, scp, minicom + +- `rpmsg-dma library `__ + +********* +EVM Setup +********* + +#. Cable Connections + + - The figure below shows some important cable connections, ports and switches. + + - Take note of the location of the "BOOTMODE" switch for SD card boot mode. + + .. figure:: /images/AM62D_evm_setup.png + :height: 600 + :width: 1000 + + - Refer `Quick Start Guide for AM62D EVM `__ for more details. + +#. Setup UART Terminal + + - First identify the UART port as enumerated on the host machine. + + - Make sure that the EVM and UART cable are connected to the UART to USB port as shown in Cable Connections. + + - In Windows, you can use the "Device Manager" to see the detected UART ports. + + - If you do not see any USB serial ports listed under "Ports (COM & LPT)", install the UART to USB driver from `FTDI `__. + + - For A53 Linux console, select UART boot port, keep other options to default and set 115200 baud rate. + +#. Setup SD card Boot Mode + + - EVM SD card boot mode setting: + + - BOOTMODE [ 8 : 15 ] (SW3) = 0100 0000 + + - BOOTMODE [ 0 : 7 ] (SW2) = 1100 0010 + +#. Audio Connections + + - Connect audio input source to J1 and J2 (PCM6240 ADC input). + + - Connect audio output to speakers/headphones via J3 and J4 (TAD5212 DAC output). + +************************* +Steps to run Edge AI demo +************************* + +There are two ways to run and visualize this demo: + +Using Web GUI +============= + +The AM62D EVM includes a web-based GUI portal that provides an out-of-box way to +run speech enhancement and audio classification demos from any browser on the same +network — no additional software installation required. + +#. Flash an SD card with the :file:`tisdk-edgeai-image-am62dxx-evm.rootfs.wic.xz` + image and follow the instructions provided at :ref:`Create SD Card ` guide. + +#. Insert the flashed SD card into the AM62D EVM, connect audio input/output and + Ethernet cable, then power on the EVM. + +#. Make sure the EVM boot mode switches are set for SD card boot as described above. + +#. The EVM should boot and display the Arago login prompt. Enter ``root`` to log in. + +#. Get the EVM IP address: + + .. code-block:: console + + root@am62dxx-evm:~# ifconfig + +#. Open a browser on your host PC and navigate to ``http://:3000`` + +#. On the home screen, click **Audio Intelligence** or select it from the left sidebar + under **Demos**. + +.. figure:: /images/am62dx_web_gui_home.png + :align: center + :width: 80% + + AM62D Web GUI — Audio Intelligence home screen + +#. Select **Speech Enhancement** (GCRN) and click the run button to start noise + reduction on the sample audio input. + +.. figure:: /images/am62dx_web_gui_speech_enhancement.png + :align: center + :width: 80% + + AM62D Web GUI — Speech Enhancement demo + +.. figure:: /images/am62dx_web_gui_speech_enhancement_result_1.png + :align: center + :width: 80% + + AM62D Web GUI — Speech Enhancement result + +#. Select **Audio Classification** (YAMNet or VGGish) and click the run button to + start audio event classification. + +.. figure:: /images/am62dx_web_gui_audio_classification.png + :align: center + :width: 80% + + AM62D Web GUI — Audio Classification demo + +.. figure:: /images/am62dx_web_gui_audio_classification_yamnet_result.png + :align: center + :width: 80% + + AM62D Web GUI — YAMNet classification result + +.. figure:: /images/am62dx_web_gui_audio_classification_vggish_result.png + :align: center + :width: 80% + + AM62D Web GUI — VGGish classification result + +Using Linux Inference Application +================================== + +The ``rpmsg_inference_example`` C++ application provides a command-line interface to +run the same inference pipelines. For full details on building and running the application, +refer to :ref:`C++ Inference Application `. + +#. Flash an SD card with the :file:`tisdk-edgeai-image-am62dxx-evm.rootfs.wic.xz` + image and boot the EVM as described above. + +#. Run speech enhancement (GCRN): + + .. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example \ + /usr/share/tvm_inference/json/pipeline_speech_enhancement.json + +#. Run audio classification (YAMNet): + + .. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example \ + /usr/share/tvm_inference/json/pipeline_audio_classification_yamnet.json + +#. Run audio classification (VGGish): + + .. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example \ + /usr/share/tvm_inference/json/pipeline_audio_classification_vggish.json + +For live ALSA input (classification pipelines only): + + .. code-block:: console + + root@am62dxx-evm:~# rpmsg_inference_example \ + /usr/share/tvm_inference/json/pipeline_audio_classification_yamnet.json \ + --device plughw:0,0 + +For raw PCM from stdin: + + .. code-block:: console + + root@am62dxx-evm:~# arecord -t raw -f S16_LE -c 1 -r 16000 | \ + rpmsg_inference_example \ + /usr/share/tvm_inference/json/pipeline_audio_classification_yamnet.json \ + --stream + +************************* +How to build Edge AI demo +************************* + +Building Edge AI image from Yocto +================================= + +- To build the Edge AI image, refer :ref:`Processor SDK - Building the SDK with Yocto ` + +Building the Linux inference application from sources +===================================================== + +#. The source code is available as part of the `rpmsg-dma `__ repository. + + .. code-block:: console + + host# git clone https://github.com/TexasInstruments/rpmsg-dma.git -b main + +#. Download and Install the AM62D Linux SDK from |__SDK_DOWNLOAD_URL__| following + the steps mentioned at :ref:`Download and Install the SDK `. + +#. Prepare the environment for cross compilation. + + .. code-block:: console + + host# source /linux-devkit/environment-setup + +#. Compile the source: + + .. code-block:: console + + [linux-devkit]:> cd + [linux-devkit]:> export NEO_TVM_PATH=/path/to/neo-tvm + [linux-devkit]:> cmake -S . -B build -DBUILD_EDGE_AI_EXAMPLE=ON \ + -DBUILD_AUDIO_OFFLOAD_EXAMPLE=OFF \ + -DBUILD_2DFFT_OFFLOAD_EXAMPLE=OFF \ + -DBUILD_SIGNAL_CHAIN_BIQUAD_EXAMPLE=OFF + [linux-devkit]:> cmake --build build + + - This builds: + + - The inference application :file:`rpmsg_inference_example` + - The model daemon :file:`tvm_model_daemon` + + - Transfer the generated files to the EVM: + + .. code-block:: console + + host# scp build/example/edge-ai/rpmsg_inference_example root@:/usr/bin/ + host# scp build/example/edge-ai/tvm_model_daemon root@:/usr/bin/ + +Building the DSP firmware from sources +====================================== + +- Refer to the `MCU+ SDK Documentation `__ +- Refer to the `Edge AI TVM + DSP Multi-Model Signal Chain `__ diff --git a/source/linux/Demo_User_Guides/index_Demos.rst b/source/linux/Demo_User_Guides/index_Demos.rst index 46f9baa6c..8ea17b266 100644 --- a/source/linux/Demo_User_Guides/index_Demos.rst +++ b/source/linux/Demo_User_Guides/index_Demos.rst @@ -18,10 +18,25 @@ The SDK supports the following Out-Of-Box demo-applications Display_Cluster_User_Guide TI_LVGL_Demo_User_Guide GUI_Frameworks_User_Guide + EV_charging_Demo_User_Guide.rst + +AM62D DSP Offload Demos +======================= + +.. toctree:: + :maxdepth: 1 + AM62D_Dsp_Offload_User_Guide.rst AM62D_2DFFT_User_Guide.rst AM62D_Cascade_Biquad_Parametric_EQ_User_Guide.rst - EV_charging_Demo_User_Guide.rst + +AM62D Edge AI Demos +=================== + +.. toctree:: + :maxdepth: 1 + + AM62D_EdgeAI_User_Guide.rst .. ifconfig:: CONFIG_sdk in ('j7_foundational')