From 568ef82fe94d24e222bae0132c0ee4421dbcf30b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobias=20J=C3=BClg?= <tobias@juelg.net>
Date: Mon, 20 Apr 2026 00:25:33 +0200
Subject: [PATCH 1/3] docs: refresh setup, ur5e, and operator guides

---
 docs/extensions/index.md            |  1 +
 docs/extensions/rcs_ur5e.md         | 49 +++++++++++++++++++++
 docs/getting_started/index.md       | 22 ++++++++--
 docs/user_guide/data_collection.md  | 67 +++++++++++++++++++++++++++++
 docs/user_guide/index.md            |  3 ++
 docs/user_guide/remote_inference.md | 53 +++++++++++++++++++++++
 docs/user_guide/teleoperation.md    | 48 +++++++++++++++++++++
 7 files changed, 239 insertions(+), 4 deletions(-)
 create mode 100644 docs/extensions/rcs_ur5e.md
 create mode 100644 docs/user_guide/data_collection.md
 create mode 100644 docs/user_guide/remote_inference.md
 create mode 100644 docs/user_guide/teleoperation.md

diff --git a/docs/extensions/index.md b/docs/extensions/index.md
index 58c0391e..2d00ab43 100644
--- a/docs/extensions/index.md
+++ b/docs/extensions/index.md
@@ -8,6 +8,7 @@ rcs_fr3
 rcs_panda
 rcs_xarm7
 rcs_so101
+rcs_ur5e
 rcs_realsense
 rcs_usb_cam
 rcs_tacto
diff --git a/docs/extensions/rcs_ur5e.md b/docs/extensions/rcs_ur5e.md
new file mode 100644
index 00000000..e0871d1a
--- /dev/null
+++ b/docs/extensions/rcs_ur5e.md
@@ -0,0 +1,49 @@
+# UR5e Extension
+
+The `rcs_ur5e` extension adds UR5e hardware support and an environment creator for Robot Control Stack.
+
+## Safety Notices
+
+- This controller does not implement compliant force control.
+- Start slowly on the teach pendant and verify the home pose before running motions.
+- Treat the extension as a low-level controller, not a safety layer.
+
+## Installation
+
+```shell
+pip install -ve extensions/rcs_ur5e
+```
+
+## Hardware Usage
+
+For direct hardware access, create a `UR5eConfig`, then build the robot with a kinematics backend.
+
+```python
+import rcs
+from rcs_ur5e.hw import UR5e, UR5eConfig
+
+cfg = UR5eConfig(ip="192.168.25.201")
+ik = rcs.common.Pin(
+    cfg.kinematic_model_path,
+    cfg.attachment_site,
+    urdf=cfg.kinematic_model_path.endswith(".urdf"),
+)
+robot = UR5e(cfg, ik)
+```
+
+For a Gymnasium-style hardware environment, use `RCSUR5eEnvCreator` from `rcs_ur5e.creators`.
+
+## Simulation Usage
+
+The simulation examples show the current setup pattern for UR5e in MuJoCo:
+
+- `examples/ur5e/ur5e_env_joint_control.py`
+- `examples/ur5e/ur5e_env_cartesian_control.py`
+
+Unlike the default FR3 simulation config, the UR5e examples set robot joints, actuators, attachment site, and kinematic model explicitly before creating the environment.
+
+## Notes
+
+- Hardware support is implemented in `extensions/rcs_ur5e/src/rcs_ur5e/hw.py`.
+- The environment creator lives in `extensions/rcs_ur5e/src/rcs_ur5e/creators.py`.
+- Use the example scripts as the source of truth if this page ever drifts.
diff --git a/docs/getting_started/index.md b/docs/getting_started/index.md
index f76229c2..d74f5cec 100644
--- a/docs/getting_started/index.md
+++ b/docs/getting_started/index.md
@@ -14,6 +14,14 @@ We build and test RCS on the latest Debian and on the latest Ubuntu LTS.
 
 2.  Create and activate Python virtual environment or conda environment:
 
+    RCS requires **Python >= 3.10**. We strongly recommend **Python 3.11** for full compatibility with all extensions (e.g., `rcs_realsense`).
+
+    ```{note}
+    - **Python 3.11** is preferred for development.
+    - **Python > 3.11**: `rcs_realsense` may have compatibility issues due to `pyrealsense2` limitations.
+    - **Python > 3.12**: The `ompl` dependency is currently unavailable on PyPI.
+    ```
+
     ```shell
     conda create -n rcs python=3.11
     conda activate rcs
@@ -53,14 +61,20 @@ from time import sleep
 import numpy as np
 
 # Load simulation scene
-simulation = sim.Sim(rcs.scenes["fr3_empty_world"].mjb)
-urdf_path = rcs.scenes["fr3_empty_world"].urdf
-ik = rcs.common.RL(str(urdf_path))
+scene = rcs.scenes["fr3_empty_world"]
+simulation = sim.Sim(scene.mjb or scene.mjcf_scene)
 
 # Configure robot
 cfg = sim.SimRobotConfig()
 cfg.add_postfix("_0")
 cfg.tcp_offset = rcs.common.Pose(rcs.common.FrankaHandTCPOffset())
+
+# Configure IK solver (using Pinocchio)
+ik = rcs.common.Pin(
+    cfg.kinematic_model_path,
+    cfg.attachment_site,
+    urdf=cfg.kinematic_model_path.endswith(".urdf"),
+)
 robot = rcs.sim.SimRobot(simulation, ik, cfg)
 
 # Configure gripper
@@ -73,7 +87,7 @@ camera_set = SimCameraSet(simulation, {})
 
 # Open GUI
 simulation.open_gui()
-sleep(5)
+sleep(2)
 
 # Step the robot 10 cm in x direction
 robot.set_cartesian_position(
diff --git a/docs/user_guide/data_collection.md b/docs/user_guide/data_collection.md
new file mode 100644
index 00000000..af54241b
--- /dev/null
+++ b/docs/user_guide/data_collection.md
@@ -0,0 +1,67 @@
+# Data Collection
+
+RCS provides tools for efficiently collecting and managing robot interaction data, primarily for Imitation Learning and Reinforcement Learning.
+
+## StorageWrapper
+
+The `StorageWrapper` is the primary tool for recording environment transitions. It is designed to be crash-safe and efficient.
+
+### Key Features
+
+- **Asynchronous Writing**: Data is written to disk in a background thread to minimize impact on the control loop.
+- **Crash-Safe**: Data is flushed in atomic batches, ensuring that most data is preserved even if the process crashes.
+- **Parquet Format**: Uses the Apache Parquet format (via `pyarrow`) for efficient storage and fast reading.
+- **Automatic Consolidation**: Small batch files are automatically merged into larger optimized files when the environment is closed.
+- **Image Compression**: RGB frames are automatically encoded as JPEGs to save space.
+
+### Usage
+
+```python
+from rcs.envs.storage_wrapper import StorageWrapper
+
+# Wrap your environment
+env = StorageWrapper(
+    env,
+    base_dir="data/my_experiment",
+    instruction="pick up the red cube",
+    always_record=False  # Only record when start_record() is called
+)
+
+# Control recording
+env.start_record()
+# ... perform tasks ...
+env.stop_record()
+
+# Close to ensure all data is flushed and consolidated
+env.close()
+```
+
+## Dataset Structure
+
+The data is organized in a directory structure partitioned by date:
+
+```text
+base_dir/
+  date=2024-05-20/
+    part-0-a1b2c3d4.parquet
+    part-1-e5f6g7h8.parquet
+  date=2024-05-21/
+    ...
+```
+
+Each Parquet file contains:
+- `obs`: The environment observation (flattened).
+- `action`: The action taken.
+- `reward`: The reward received.
+- `success`: Boolean indicating task success.
+- `instruction`: The text instruction for the task.
+- `timestamp`: Unix timestamp of the transition.
+- `uuid`: A unique identifier for the episode.
+
+## Consolidating Data
+
+If a script exits unexpectedly and consolidation doesn't run, you can manually consolidate the fragmented files using the `StorageWrapper.consolidate` static method or the RCS CLI:
+
+```shell
+python -m rcs consolidate data/my_experiment
+```
diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md
index 739caecc..4f321207 100644
--- a/docs/user_guide/index.md
+++ b/docs/user_guide/index.md
@@ -8,4 +8,7 @@ The User Guide provides in-depth information about the core concepts and compone
 architecture
 gym_interface
 low_level_api
+teleoperation
+data_collection
+remote_inference
 ```
diff --git a/docs/user_guide/remote_inference.md b/docs/user_guide/remote_inference.md
new file mode 100644
index 00000000..01cb71ec
--- /dev/null
+++ b/docs/user_guide/remote_inference.md
@@ -0,0 +1,53 @@
+# Remote Inference
+
+RCS includes a lightweight RPC (Remote Procedure Call) layer based on [RPyC](https://rpyc.readthedocs.io/). This allows you to run an RCS environment on one machine (e.g., a machine connected to robot hardware) and control it or run inference from another machine (e.g., a powerful GPU server).
+
+## Overview
+
+The RPC system consists of two main components:
+- **`RcsServer`**: Wraps an existing RCS environment and exposes it over the network.
+- **`RcsClient`**: A Gymnasium-compatible environment that forwards all calls to a remote `RcsServer`.
+
+## Usage
+
+### Starting the Server
+
+The server machine should be the one physically connected to the robot or running the simulation.
+
+```python
+from rcs.envs.creators import SimEnvCreator
+from rcs.rpc.server import RcsServer
+
+# Create your environment
+env = SimEnvCreator()(...)
+
+# Start the RPC server
+server = RcsServer(env, port=50051)
+server.start()
+```
+
+### Connecting with the Client
+
+The client machine runs your control logic or neural network inference.
+
+```python
+from rcs.rpc.client import RcsClient
+
+# Connect to the remote server
+client = RcsClient(host="robot-machine-ip", port=50051)
+
+# Use it like a local Gymnasium environment
+obs, info = client.reset()
+action = model.predict(obs)
+obs, reward, terminated, truncated, info = client.step(action)
+```
+
+## Advantages
+
+- **Hardware Isolation**: Keep your expensive GPU server away from the robot workspace.
+- **Resource Management**: Run heavy inference on a dedicated machine while the robot control machine handles low-level loops.
+- **Flexibility**: Easily switch between local and remote environments by just changing the environment initialization.
+
+## Examples
+
+See `examples/rpc_server_client/` for a complete working example of a server and client setup.
diff --git a/docs/user_guide/teleoperation.md b/docs/user_guide/teleoperation.md
new file mode 100644
index 00000000..8e111d89
--- /dev/null
+++ b/docs/user_guide/teleoperation.md
@@ -0,0 +1,48 @@
+# Teleoperation
+
+RCS supports teleoperation using various devices, including VR headsets (Meta Quest 3) and specialized hardware like GELLO.
+
+## VR Teleoperation (Meta Quest 3)
+
+The [IRIS platform](https://intuitive-robots.github.io/iris-project-page/index.html) is used to stream controller poses from the Meta Quest 3 to RCS.
+
+### Setup
+
+1.  **Install IRIS**: Follow the instructions on the [IRIS Meta Quest repository](https://github.com/intuitive-robots/IRIS-Meta-Quest3).
+2.  **Install SimPub**: The IRIS Python client.
+    ```shell
+    pip install -r examples/teleop/requirements.txt
+    ```
+3.  **Calibrate**: Run the alignment script to map the VR coordinate system to the robot's base frame.
+    ```shell
+    python examples/teleop/quest_align_frame.py
+    ```
+
+### Running Teleoperation
+
+The `franka.py` example demonstrates teleoperating a Franka FR3 (sim or hardware). It uses the `RelativeActionSpace` wrapper to apply delta movements from the VR controllers.
+
+```python
+# From examples/teleop/franka.py
+# The script configures the environment to follow controller poses
+# and records data using the StorageWrapper.
+```
+
+## Hardware Teleoperation (GELLO)
+
+GELLO is a 3D-printed, low-cost teleoperation device. RCS supports GELLO for various robots.
+
+### Setup
+
+1.  Connect the GELLO device via USB.
+2.  Install dependencies:
+    ```shell
+    pip install -r examples/teleop/requirements.txt
+    ```
+3.  Configure the `GelloConfig` in your teleoperation script with the appropriate USB IDs.
+
+## Key Concepts
+
+- **Relative Control**: Teleoperation typically uses delta-movements. The `RelativeActionSpace` wrapper is used to translate absolute device poses into relative robot movements.
+- **Visual Feedback**: When teleoperating in simulation, the GUI provides real-time visual feedback. For hardware, cameras (e.g., RealSense) can be used to provide a first-person view.
+- **Data Collection**: Teleoperation is often used to collect expert demonstrations for Imitation Learning. This is integrated via the `StorageWrapper`.

From 07d42716ab3a44fe705657429848572ad1d1e566 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobias=20J=C3=BClg?= <tobias@juelg.net>
Date: Mon, 20 Apr 2026 08:29:52 +0200
Subject: [PATCH 2/3] docs: add 3D conventions guide

---
 docs/user_guide/conventions.md | 167 +++++++++++++++++++++++++++++++++
 docs/user_guide/index.md       |   1 +
 2 files changed, 168 insertions(+)
 create mode 100644 docs/user_guide/conventions.md

diff --git a/docs/user_guide/conventions.md b/docs/user_guide/conventions.md
new file mode 100644
index 00000000..8595c9d6
--- /dev/null
+++ b/docs/user_guide/conventions.md
@@ -0,0 +1,167 @@
+# Coordinate and Pose Conventions
+
+This page collects the main 3D conventions used across RCS. It is meant as a quick reference when working with kinematics, environments, simulation objects, and teleoperation.
+
+## Frames
+
+### World frame
+
+In simulation, scenes and objects live in a global **world frame**.
+
+The core robot API exposes explicit conversions between world and robot coordinates:
+
+- `Robot.get_base_pose_in_world_coordinates()`
+- `Robot.to_pose_in_world_coordinates(...)`
+- `Robot.to_pose_in_robot_coordinates(...)`
+
+### Robot base frame
+
+Kinematics and low-level robot poses are expressed in the robot's **base frame** (also called robot coordinates).
+
+This is the frame assumed by the kinematics backends, for example in `extensions/rcs_robotics_library/src/pybind/RL.h`:
+
+- `inverse(...)`: `pose is assumed to be in the robots coordinate frame`
+- `forward(...)`: `pose is assumed to be in the robots coordinate frame`
+
+The Franka hardware code uses the same convention and explicitly refers to the end-effector pose in the **base frame**.
+
+### End-effector frame: `attachment_site`
+
+Each robot config defines an `attachment_site`. This is the end-effector frame used by the kinematics stack.
+
+Common examples in the repository are:
+
+- `attachment_site_0` for FR3 / Panda
+- `attachment_site` for UR5e / XArm7
+- `gripper` in the SO101 examples
+
+If you are unsure which frame a robot uses, check its config or the relevant example script.
+
+### Tool frame: `tcp_offset`
+
+`tcp_offset` is applied on top of the attachment site to define the actual tool center point (TCP) used by motion commands and IK.
+
+In other words:
+
+- `attachment_site` = default end-effector frame from the model
+- `tcp_offset` = additional transform from that frame to the tool you want to control
+
+## Pose representations
+
+RCS uses several pose encodings. The important ones are:
+
+### `Pose`
+
+`rcs.common.Pose` is the main transform type. It supports construction from:
+
+- translation only
+- quaternion + translation
+- `RPY` + translation
+- rotation matrix + translation
+
+### Quaternion order
+
+Within RCS, quaternions are stored in **xyzw** order.
+
+This is visible in two places:
+
+- `python/tests/test_common.py` checks that the identity quaternion is `[0, 0, 0, 1]`
+- `python/rcs/envs/sim.py` comments `rotation_q()` as `# xyzw format`
+
+So the convention is:
+
+```text
+[qx, qy, qz, qw]
+```
+
+### `tquat`
+
+`tquat` means translation plus quaternion and is used by the environment API.
+
+The value layout is:
+
+```text
+[x, y, z, qx, qy, qz, qw]
+```
+
+This follows directly from `python/rcs/envs/base.py`, where `tquat` is built as:
+
+```python
+np.concatenate([
+    pose.translation(),
+    pose.rotation_q(),
+])
+```
+
+### `xyzrpy`
+
+`xyzrpy` is the translation plus roll-pitch-yaw representation used by the environment API.
+
+The value layout is:
+
+```text
+[x, y, z, roll, pitch, yaw]
+```
+
+The `RPY` type in `python/rcs/_core/common.pyi` exposes the fields in exactly that order:
+
+- `roll`
+- `pitch`
+- `yaw`
+
+RCS examples and environment limits use `np.deg2rad(...)`, so angles are expected in **radians** unless explicitly documented otherwise.
+
+### Rotation vector / `rotvec`
+
+Some hardware integrations, notably UR, also use a 6D rotation-vector pose:
+
+```text
+[x, y, z, rx, ry, rz]
+```
+
+You can see this in `extensions/rcs_ur5e/src/rcs_ur5e/hw.py`, where `common.RotVec(...).as_quaternion_vector()` is converted into an RCS `Pose`, and `Pose.rotvec()` is sent back to the robot.
+
+## MuJoCo caveat: free-joint quaternions use `wxyz`
+
+A common source of confusion is that **RCS uses `xyzw`**, but MuJoCo free-joint `qpos` stores the quaternion as **`wxyz`**.
+
+RCS already handles this conversion where needed. For example, `python/rcs/envs/sim.py` explicitly reorders the quaternion when writing directly into MuJoCo joint state:
+
+```python
+quat = self.init_object_pose.rotation_q()  # xyzw format
+...
+[self.x, self.y, self.z, quat[3], quat[0], quat[1], quat[2]]
+```
+
+So:
+
+- **RCS `Pose` / env API**: `xyzw`
+- **MuJoCo free-joint `qpos`**: `wxyz`
+
+If you manipulate MuJoCo state directly, convert between the two explicitly.
+
+## Axis convention
+
+RCS generally works with robot-local base coordinates, and the standard Franka teleoperation setup documents the expected axis orientation as:
+
+```text
+x = front
+y = left
+z = up
+```
+
+This comes from `examples/teleop/README.md`, which also notes that the Quest alignment step should match the robot's base frame.
+
+If you are integrating a new robot, make sure its model, `attachment_site`, and teleop calibration all agree on the same base-frame orientation.
+
+## Practical checklist
+
+When something looks wrong, check these first:
+
+1. Are you working in **world frame** or **robot/base frame**?
+2. Is your end-effector frame the correct `attachment_site`?
+3. Did you apply the right `tcp_offset`?
+4. Are your quaternions **xyzw** in RCS?
+5. Are you accidentally feeding **MuJoCo `wxyz`** into an RCS API?
+6. Are your RPY values in **radians**?
+7. Does your teleop or calibration frame match the robot base axes?
diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md
index 4f321207..290c9ab8 100644
--- a/docs/user_guide/index.md
+++ b/docs/user_guide/index.md
@@ -8,6 +8,7 @@ The User Guide provides in-depth information about the core concepts and compone
 architecture
 gym_interface
 low_level_api
+conventions
 teleoperation
 data_collection
 remote_inference

From b88e7acb273097dd4534ae99bb6378f230adbc78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobias=20J=C3=BClg?= <tobias@juelg.net>
Date: Thu, 23 Apr 2026 08:48:34 +0200
Subject: [PATCH 3/3] docs: refresh remote inference example

---
 docs/user_guide/remote_inference.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/user_guide/remote_inference.md b/docs/user_guide/remote_inference.md
index 01cb71ec..732fb728 100644
--- a/docs/user_guide/remote_inference.md
+++ b/docs/user_guide/remote_inference.md
@@ -15,11 +15,12 @@ The RPC system consists of two main components:
 The server machine should be the one physically connected to the robot or running the simulation.
 
 ```python
-from rcs.envs.creators import SimEnvCreator
+from rcs.envs.configs import EmptyWorldFR3
 from rcs.rpc.server import RcsServer
 
 # Create your environment
-env = SimEnvCreator()(...)
+scene = EmptyWorldFR3()
+env = scene.create_env(scene.config())
 
 # Start the RPC server
 server = RcsServer(env, port=50051)