diff --git a/.github/workflows/uav_nn_perception_navigation.yml b/.github/workflows/uav_nn_perception_navigation.yml new file mode 100644 index 0000000000..947c96467c --- /dev/null +++ b/.github/workflows/uav_nn_perception_navigation.yml @@ -0,0 +1,52 @@ +name: UAV neural perception navigation + +on: + push: + paths: + - 'src/air/uav_nn_perception_navigation/**' + - 'docs/air/uav_nn_perception_navigation/**' + - '.github/workflows/uav_nn_perception_navigation.yml' + pull_request: + paths: + - 'src/air/uav_nn_perception_navigation/**' + - 'docs/air/uav_nn_perception_navigation/**' + - '.github/workflows/uav_nn_perception_navigation.yml' + workflow_dispatch: + +jobs: + build-and-test: + runs-on: ubuntu-22.04 + container: ros:noetic-ros-base + steps: + - uses: actions/checkout@v4 + - name: Install ROS and test dependencies + run: | + apt-get update -qq + apt-get install -y -qq --no-install-recommends \ + python3-numpy python3-pytest ros-noetic-tf ros-noetic-tf2-ros + - name: Build package with catkin + shell: bash + run: | + source /opt/ros/noetic/setup.bash + mkdir -p /tmp/uav_catkin_ws/src + ln -s "$GITHUB_WORKSPACE/src/air/uav_nn_perception_navigation" \ + /tmp/uav_catkin_ws/src/uav_nn_perception_navigation + cd /tmp/uav_catkin_ws + catkin_make + - name: Run module tests + shell: bash + run: | + source /opt/ros/noetic/setup.bash + python3 -m pytest src/air/uav_nn_perception_navigation/test -q + + docs: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install MkDocs dependencies + run: python -m pip install -r requirements-docs.txt + - name: Build documentation + run: python -m mkdocs build --clean --site-dir /tmp/uav_docs_site diff --git a/docs/air/uav_nn_perception_navigation/index.md b/docs/air/uav_nn_perception_navigation/index.md index 2b71411acf..7057637879 100644 --- a/docs/air/uav_nn_perception_navigation/index.md +++ b/docs/air/uav_nn_perception_navigation/index.md @@ -688,7 +688,7 @@ $$v_{rep}=K_{rep}\left(1-\frac{d}{6.0}\right),\qquad K_{rep}=4.0$$ ### 9.6 端到端 camera_perception.py 与 end2end_node.py -`camera_perception.py` 负责取图。它一次 `simGetImages` 同时请求两张:`ImageType.Scene`(uint8 RGB)和 `ImageType.DepthPlanar`(32 位浮点、单位为米),省掉两次往返。发布时不用 `cv_bridge`,而是自己构造 `sensor_msgs/Image`,源码注释给出的理由是减少环境依赖。`step` 按行长乘通道字节数现场算:三维数组是 `width * 3 * itemsize`,二维数组是 `width * itemsize`。深度图只在 `~publish_depth` 为真时请求与发布,编码为 `32FC1`,RGB 为 `rgb8`,两者的 `frame_id` 都是 `base_link`。 +`camera_perception.py` 负责取图。它一次 `simGetImages` 同时请求两张:`ImageType.Scene`(uint8 BGR 或 BGRA)和 `ImageType.DepthPlanar`(32 位浮点、单位为米),省掉两次往返。场景图若为四通道则丢弃 alpha,保留训练权重使用的 BGR 顺序;发布的 `sensor_msgs/Image` 编码为 `bgr8`,深度图为 `32FC1`。发布时不用 `cv_bridge`,`step` 按每行实际字节数计算,两者的 `frame_id` 都是 `base_link`。 `end2end_node.py` 把图像直接映射成速度指令,中间没有任何人工规则。网络定义与下采样函数都从 `train_end2end` 导入,保证结构不会漂移:输入先降到 `IMG_H, IMG_W = 36, 64`,三次卷积每次空间尺寸减半,最后接两层全连接: @@ -713,7 +713,7 @@ $$I_{out}(i,j,c)=\frac{1}{f_hf_w}\sum_{u=0}^{f_h-1}\sum_{v=0}^{f_w-1}I(i f_h+u,\ 发布的指令做三重限幅:线速度 `np.clip` 到 ±`max_speed`(默认 3.0),偏航角速率截断到 ±1.0,高度通道仍是 0.8 倍比例控制并限在 ±1.5。这与 9.3 的限幅方式一致,因为两者共用同一个桥接节点。 -两处坑。其一是 `image_cb` 只接受 `encoding == 'rgb8'` 的消息,其它编码静默丢弃,若改用了别的图像源会表现为"节点一直不动作"。其二是 `downsample` 在图像小于目标尺寸时返回 `None`,回调里只在非空时才覆盖 `self.image`,否则保留上一帧;`run` 循环里图像或里程计任一为空就不发布,注释说明是为了不干扰起飞。 +两处坑。其一是 `image_cb` 接受 `bgr8`,也兼容 `rgb8` 并转换为模型训练时的 BGR 顺序;其它编码会被丢弃。其二是 `downsample` 在图像小于目标尺寸时返回 `None`,回调里只在非空时才覆盖 `self.image`,否则保留上一帧;`run` 循环里图像或里程计任一为空就不发布,注释说明是为了不干扰起飞。 ### 9.7 统一入口 main.py diff --git a/src/air/uav_nn_perception_navigation/CMakeLists.txt b/src/air/uav_nn_perception_navigation/CMakeLists.txt index 030b94752f..aa3d844b99 100644 --- a/src/air/uav_nn_perception_navigation/CMakeLists.txt +++ b/src/air/uav_nn_perception_navigation/CMakeLists.txt @@ -36,12 +36,14 @@ install(FILES scripts/safety_shield.py scripts/train_control.py scripts/train_end2end.py + scripts/scene_image.py DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} ) if(CATKIN_ENABLE_TESTING) catkin_add_nosetests(test/test_uav_features.py) catkin_add_nosetests(test/test_safety_shield.py) + catkin_add_nosetests(test/test_scene_image.py) endif() install(DIRECTORY launch config models diff --git a/src/air/uav_nn_perception_navigation/scripts/camera_perception.py b/src/air/uav_nn_perception_navigation/scripts/camera_perception.py index 53d15f3e1b..6557345f2c 100755 --- a/src/air/uav_nn_perception_navigation/scripts/camera_perception.py +++ b/src/air/uav_nn_perception_navigation/scripts/camera_perception.py @@ -4,7 +4,7 @@ 相机感知节点(任务 4 的感知部分) 从 AirSim 读取机载相机图像,发布标准 ROS 图像话题: - /uav/image_raw sensor_msgs/Image rgb8 + /uav/image_raw sensor_msgs/Image bgr8 /uav/depth sensor_msgs/Image 32FC1(米) 不依赖 cv_bridge,直接构造消息,减少环境依赖。 @@ -17,11 +17,24 @@ import rospy from sensor_msgs.msg import Image +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +if _SCRIPT_DIR not in sys.path: + sys.path.insert(0, _SCRIPT_DIR) +try: + import rospkg + _source_scripts = os.path.join( + rospkg.RosPack().get_path('uav_nn_perception_navigation'), 'scripts') + if _source_scripts not in sys.path: + sys.path.insert(0, _source_scripts) +except Exception: + pass + _client_path = os.environ.get('AIRSIM_PYTHON_CLIENT') if _client_path and _client_path not in sys.path: sys.path.insert(0, _client_path) import airsim # noqa: E402 +from scene_image import decode_scene_bgr # noqa: E402 def to_image_msg(arr, encoding, stamp): @@ -67,9 +80,8 @@ def run(self): responses = self.client.simGetImages(reqs, vehicle_name=self.vehicle) rgb = responses[0] - img = np.frombuffer(rgb.image_data_uint8, dtype=np.uint8) - img = img.reshape(rgb.height, rgb.width, 3) - self.rgb_pub.publish(to_image_msg(img, 'rgb8', stamp)) + img = decode_scene_bgr(rgb.image_data_uint8, rgb.height, rgb.width) + self.rgb_pub.publish(to_image_msg(img, 'bgr8', stamp)) if self.publish_depth and len(responses) > 1: depth = responses[1] diff --git a/src/air/uav_nn_perception_navigation/scripts/end2end_node.py b/src/air/uav_nn_perception_navigation/scripts/end2end_node.py index bd0dde3082..66f7eea4b9 100755 --- a/src/air/uav_nn_perception_navigation/scripts/end2end_node.py +++ b/src/air/uav_nn_perception_navigation/scripts/end2end_node.py @@ -21,6 +21,14 @@ _SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) if _SCRIPT_DIR not in sys.path: sys.path.insert(0, _SCRIPT_DIR) +try: + import rospkg + _source_scripts = os.path.join( + rospkg.RosPack().get_path('uav_nn_perception_navigation'), 'scripts') + if _source_scripts not in sys.path: + sys.path.insert(0, _source_scripts) +except Exception: + pass from train_end2end import IMG_H, IMG_W, End2EndNet, downsample # noqa: E402 @@ -65,10 +73,12 @@ def __init__(self): self.pub = rospy.Publisher('/uav/cmd_vel', Twist, queue_size=1) def image_cb(self, msg): - if msg.encoding != 'rgb8': + if msg.encoding not in ('bgr8', 'rgb8'): return arr = np.frombuffer(msg.data, dtype=np.uint8) arr = arr.reshape(msg.height, msg.width, 3) + if msg.encoding == 'rgb8': + arr = arr[:, :, ::-1] # old model expects BGR small = downsample(arr, IMG_H, IMG_W) if small is not None: self.image = small.astype(np.float32) / 255.0 diff --git a/src/air/uav_nn_perception_navigation/scripts/scene_image.py b/src/air/uav_nn_perception_navigation/scripts/scene_image.py new file mode 100644 index 0000000000..5f6e291c4a --- /dev/null +++ b/src/air/uav_nn_perception_navigation/scripts/scene_image.py @@ -0,0 +1,20 @@ +"""Decode AirSim uncompressed Scene images for the trained BGR model.""" + +import numpy as np + + +def decode_scene_bgr(data, height, width): + """Return an HxWx3 BGR image from AirSim BGR or BGRA bytes. + + Existing end-to-end weights were trained on BGR channel order, so dropping + BGRA's alpha preserves the model input instead of silently swapping colors. + """ + if height <= 0 or width <= 0: + raise ValueError('Scene image dimensions must be positive') + raw = np.frombuffer(data, dtype=np.uint8) + pixels = height * width + if raw.size == pixels * 3: + return raw.reshape(height, width, 3) + if raw.size == pixels * 4: + return np.ascontiguousarray(raw.reshape(height, width, 4)[:, :, :3]) + raise ValueError('Scene image must contain 3 or 4 channels per pixel') diff --git a/src/air/uav_nn_perception_navigation/scripts/train_end2end.py b/src/air/uav_nn_perception_navigation/scripts/train_end2end.py index f1ec2218b1..11e7257596 100644 --- a/src/air/uav_nn_perception_navigation/scripts/train_end2end.py +++ b/src/air/uav_nn_perception_navigation/scripts/train_end2end.py @@ -29,6 +29,7 @@ from train_control import ACTION_SCALE, expert_action, place_free # noqa: E402 from uav_features import build_feature_vector, yaw_from_quaternion # noqa: E402 +from scene_image import decode_scene_bgr # noqa: E402 _client_path = os.environ.get('AIRSIM_PYTHON_CLIENT') if _client_path and _client_path not in sys.path: @@ -96,8 +97,7 @@ def collect(client, episodes, steps, rng, altitude=-4.0): resp = client.simGetImages( [airsim.ImageRequest('front_center', airsim.ImageType.Scene, False, False)])[0] - raw = np.frombuffer(resp.image_data_uint8, dtype=np.uint8) - img = raw.reshape(resp.height, resp.width, 3) + img = decode_scene_bgr(resp.image_data_uint8, resp.height, resp.width) small = downsample(img) if small is None: continue diff --git a/src/air/uav_nn_perception_navigation/test/test_scene_image.py b/src/air/uav_nn_perception_navigation/test/test_scene_image.py new file mode 100644 index 0000000000..474ce5ce86 --- /dev/null +++ b/src/air/uav_nn_perception_navigation/test/test_scene_image.py @@ -0,0 +1,33 @@ +"""AirSim Scene decoding must accept both supported byte layouts.""" + +import os +import sys +import unittest + +import numpy as np + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'scripts')) +from scene_image import decode_scene_bgr # noqa: E402 + + +class TestSceneImage(unittest.TestCase): + def test_bgr_is_preserved(self): + image = decode_scene_bgr(bytes([10, 20, 30, 40, 50, 60]), 1, 2) + np.testing.assert_array_equal(image, [[[10, 20, 30], [40, 50, 60]]]) + + def test_bgra_drops_alpha(self): + image = decode_scene_bgr(bytes([10, 20, 30, 255, 40, 50, 60, 0]), 1, 2) + np.testing.assert_array_equal(image, [[[10, 20, 30], [40, 50, 60]]]) + self.assertTrue(image.flags.c_contiguous) + + def test_invalid_shape_is_rejected(self): + with self.assertRaises(ValueError): + decode_scene_bgr(bytes([1, 2]), 1, 1) + + def test_empty_dimensions_are_rejected(self): + with self.assertRaises(ValueError): + decode_scene_bgr(b'', 0, 2) + + +if __name__ == '__main__': + unittest.main()