Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions .github/workflows/uav_nn_perception_navigation.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
name: UAV neural perception navigation

on:
push:
paths:
- 'src/air/uav_nn_perception_navigation/**'
- 'docs/air/uav_nn_perception_navigation/**'
- '.github/workflows/uav_nn_perception_navigation.yml'
pull_request:
paths:
- 'src/air/uav_nn_perception_navigation/**'
- 'docs/air/uav_nn_perception_navigation/**'
- '.github/workflows/uav_nn_perception_navigation.yml'
workflow_dispatch:

jobs:
build-and-test:
runs-on: ubuntu-22.04
container: ros:noetic-ros-base
steps:
- uses: actions/checkout@v4
- name: Install ROS and test dependencies
run: |
apt-get update -qq
apt-get install -y -qq --no-install-recommends \
python3-numpy python3-pytest ros-noetic-tf ros-noetic-tf2-ros
- name: Build package with catkin
shell: bash
run: |
source /opt/ros/noetic/setup.bash
mkdir -p /tmp/uav_catkin_ws/src
ln -s "$GITHUB_WORKSPACE/src/air/uav_nn_perception_navigation" \
/tmp/uav_catkin_ws/src/uav_nn_perception_navigation
cd /tmp/uav_catkin_ws
catkin_make
- name: Run module tests
shell: bash
run: |
source /opt/ros/noetic/setup.bash
python3 -m pytest src/air/uav_nn_perception_navigation/test -q

docs:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install MkDocs dependencies
run: python -m pip install -r requirements-docs.txt
- name: Build documentation
run: python -m mkdocs build --clean --site-dir /tmp/uav_docs_site
4 changes: 2 additions & 2 deletions docs/air/uav_nn_perception_navigation/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -688,7 +688,7 @@ $$v_{rep}=K_{rep}\left(1-\frac{d}{6.0}\right),\qquad K_{rep}=4.0$$

### 9.6 端到端 camera_perception.py 与 end2end_node.py

`camera_perception.py` 负责取图。它一次 `simGetImages` 同时请求两张:`ImageType.Scene`(uint8 RGB)和 `ImageType.DepthPlanar`(32 位浮点、单位为米),省掉两次往返。发布时不用 `cv_bridge`,而是自己构造 `sensor_msgs/Image`,源码注释给出的理由是减少环境依赖。`step` 按行长乘通道字节数现场算:三维数组是 `width * 3 * itemsize`,二维数组是 `width * itemsize`。深度图只在 `~publish_depth` 为真时请求与发布,编码为 `32FC1`,RGB 为 `rgb8`,两者的 `frame_id` 都是 `base_link`。
`camera_perception.py` 负责取图。它一次 `simGetImages` 同时请求两张:`ImageType.Scene`(uint8 BGR 或 BGRA)和 `ImageType.DepthPlanar`(32 位浮点、单位为米),省掉两次往返。场景图若为四通道则丢弃 alpha,保留训练权重使用的 BGR 顺序;发布的 `sensor_msgs/Image` 编码为 `bgr8`,深度图为 `32FC1`。发布时不用 `cv_bridge`,`step` 按每行实际字节数计算,两者的 `frame_id` 都是 `base_link`。

`end2end_node.py` 把图像直接映射成速度指令,中间没有任何人工规则。网络定义与下采样函数都从 `train_end2end` 导入,保证结构不会漂移:输入先降到 `IMG_H, IMG_W = 36, 64`,三次卷积每次空间尺寸减半,最后接两层全连接:

Expand All @@ -713,7 +713,7 @@ $$I_{out}(i,j,c)=\frac{1}{f_hf_w}\sum_{u=0}^{f_h-1}\sum_{v=0}^{f_w-1}I(i f_h+u,\

发布的指令做三重限幅:线速度 `np.clip` 到 ±`max_speed`(默认 3.0),偏航角速率截断到 ±1.0,高度通道仍是 0.8 倍比例控制并限在 ±1.5。这与 9.3 的限幅方式一致,因为两者共用同一个桥接节点。

两处坑。其一是 `image_cb` 只接受 `encoding == 'rgb8'` 的消息,其它编码静默丢弃,若改用了别的图像源会表现为"节点一直不动作"。其二是 `downsample` 在图像小于目标尺寸时返回 `None`,回调里只在非空时才覆盖 `self.image`,否则保留上一帧;`run` 循环里图像或里程计任一为空就不发布,注释说明是为了不干扰起飞。
两处坑。其一是 `image_cb` 接受 `bgr8`,也兼容 `rgb8` 并转换为模型训练时的 BGR 顺序;其它编码会被丢弃。其二是 `downsample` 在图像小于目标尺寸时返回 `None`,回调里只在非空时才覆盖 `self.image`,否则保留上一帧;`run` 循环里图像或里程计任一为空就不发布,注释说明是为了不干扰起飞。

### 9.7 统一入口 main.py

Expand Down
2 changes: 2 additions & 0 deletions src/air/uav_nn_perception_navigation/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,14 @@ install(FILES
scripts/safety_shield.py
scripts/train_control.py
scripts/train_end2end.py
scripts/scene_image.py
DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
)

if(CATKIN_ENABLE_TESTING)
catkin_add_nosetests(test/test_uav_features.py)
catkin_add_nosetests(test/test_safety_shield.py)
catkin_add_nosetests(test/test_scene_image.py)
endif()

install(DIRECTORY launch config models
Expand Down
20 changes: 16 additions & 4 deletions src/air/uav_nn_perception_navigation/scripts/camera_perception.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
相机感知节点(任务 4 的感知部分)

从 AirSim 读取机载相机图像,发布标准 ROS 图像话题:
/uav/image_raw sensor_msgs/Image rgb8
/uav/image_raw sensor_msgs/Image bgr8
/uav/depth sensor_msgs/Image 32FC1(米)

不依赖 cv_bridge,直接构造消息,减少环境依赖。
Expand All @@ -17,11 +17,24 @@
import rospy
from sensor_msgs.msg import Image

_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
if _SCRIPT_DIR not in sys.path:
sys.path.insert(0, _SCRIPT_DIR)
try:
import rospkg
_source_scripts = os.path.join(
rospkg.RosPack().get_path('uav_nn_perception_navigation'), 'scripts')
if _source_scripts not in sys.path:
sys.path.insert(0, _source_scripts)
except Exception:
pass

_client_path = os.environ.get('AIRSIM_PYTHON_CLIENT')
if _client_path and _client_path not in sys.path:
sys.path.insert(0, _client_path)

import airsim # noqa: E402
from scene_image import decode_scene_bgr # noqa: E402


def to_image_msg(arr, encoding, stamp):
Expand Down Expand Up @@ -67,9 +80,8 @@ def run(self):
responses = self.client.simGetImages(reqs, vehicle_name=self.vehicle)

rgb = responses[0]
img = np.frombuffer(rgb.image_data_uint8, dtype=np.uint8)
img = img.reshape(rgb.height, rgb.width, 3)
self.rgb_pub.publish(to_image_msg(img, 'rgb8', stamp))
img = decode_scene_bgr(rgb.image_data_uint8, rgb.height, rgb.width)
self.rgb_pub.publish(to_image_msg(img, 'bgr8', stamp))

if self.publish_depth and len(responses) > 1:
depth = responses[1]
Expand Down
12 changes: 11 additions & 1 deletion src/air/uav_nn_perception_navigation/scripts/end2end_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,14 @@
_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
if _SCRIPT_DIR not in sys.path:
sys.path.insert(0, _SCRIPT_DIR)
try:
import rospkg
_source_scripts = os.path.join(
rospkg.RosPack().get_path('uav_nn_perception_navigation'), 'scripts')
if _source_scripts not in sys.path:
sys.path.insert(0, _source_scripts)
except Exception:
pass

from train_end2end import IMG_H, IMG_W, End2EndNet, downsample # noqa: E402

Expand Down Expand Up @@ -65,10 +73,12 @@ def __init__(self):
self.pub = rospy.Publisher('/uav/cmd_vel', Twist, queue_size=1)

def image_cb(self, msg):
if msg.encoding != 'rgb8':
if msg.encoding not in ('bgr8', 'rgb8'):
return
arr = np.frombuffer(msg.data, dtype=np.uint8)
arr = arr.reshape(msg.height, msg.width, 3)
if msg.encoding == 'rgb8':
arr = arr[:, :, ::-1] # old model expects BGR
small = downsample(arr, IMG_H, IMG_W)
if small is not None:
self.image = small.astype(np.float32) / 255.0
Expand Down
20 changes: 20 additions & 0 deletions src/air/uav_nn_perception_navigation/scripts/scene_image.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
"""Decode AirSim uncompressed Scene images for the trained BGR model."""

import numpy as np


def decode_scene_bgr(data, height, width):
"""Return an HxWx3 BGR image from AirSim BGR or BGRA bytes.

Existing end-to-end weights were trained on BGR channel order, so dropping
BGRA's alpha preserves the model input instead of silently swapping colors.
"""
if height <= 0 or width <= 0:
raise ValueError('Scene image dimensions must be positive')
raw = np.frombuffer(data, dtype=np.uint8)
pixels = height * width
if raw.size == pixels * 3:
return raw.reshape(height, width, 3)
if raw.size == pixels * 4:
return np.ascontiguousarray(raw.reshape(height, width, 4)[:, :, :3])
raise ValueError('Scene image must contain 3 or 4 channels per pixel')
4 changes: 2 additions & 2 deletions src/air/uav_nn_perception_navigation/scripts/train_end2end.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

from train_control import ACTION_SCALE, expert_action, place_free # noqa: E402
from uav_features import build_feature_vector, yaw_from_quaternion # noqa: E402
from scene_image import decode_scene_bgr # noqa: E402

_client_path = os.environ.get('AIRSIM_PYTHON_CLIENT')
if _client_path and _client_path not in sys.path:
Expand Down Expand Up @@ -96,8 +97,7 @@ def collect(client, episodes, steps, rng, altitude=-4.0):
resp = client.simGetImages(
[airsim.ImageRequest('front_center', airsim.ImageType.Scene,
False, False)])[0]
raw = np.frombuffer(resp.image_data_uint8, dtype=np.uint8)
img = raw.reshape(resp.height, resp.width, 3)
img = decode_scene_bgr(resp.image_data_uint8, resp.height, resp.width)
small = downsample(img)
if small is None:
continue
Expand Down
33 changes: 33 additions & 0 deletions src/air/uav_nn_perception_navigation/test/test_scene_image.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
"""AirSim Scene decoding must accept both supported byte layouts."""

import os
import sys
import unittest

import numpy as np

sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'scripts'))
from scene_image import decode_scene_bgr # noqa: E402


class TestSceneImage(unittest.TestCase):
def test_bgr_is_preserved(self):
image = decode_scene_bgr(bytes([10, 20, 30, 40, 50, 60]), 1, 2)
np.testing.assert_array_equal(image, [[[10, 20, 30], [40, 50, 60]]])

def test_bgra_drops_alpha(self):
image = decode_scene_bgr(bytes([10, 20, 30, 255, 40, 50, 60, 0]), 1, 2)
np.testing.assert_array_equal(image, [[[10, 20, 30], [40, 50, 60]]])
self.assertTrue(image.flags.c_contiguous)

def test_invalid_shape_is_rejected(self):
with self.assertRaises(ValueError):
decode_scene_bgr(bytes([1, 2]), 1, 1)

def test_empty_dimensions_are_rejected(self):
with self.assertRaises(ValueError):
decode_scene_bgr(b'', 0, 2)


if __name__ == '__main__':
unittest.main()
Loading