Rust I18N Rewrite

src/rust/cxx/PATCHES.md

@@ -13,5 +13,14 @@ The in-tree fork contains these major behavioral changes:
   `kj::Maybe`, `kj::Date`, exceptions, promises, and Rust futures.
 - **`__WORKERD_CXX__`.** This preprocessor definition identifies the fork even though its crate and
   generated symbol names remain compatible with cxx.
+- **`c_char16`.** An additional builtin type, spelled `char16_t` in C++, for UTF-16 data. `u16`
+  means `uint16_t`, which C++ mangles and overloads separately from `char16_t`, so a bridge that
+  must name `char16_t` cannot use `u16`. The Rust side is an alias for `u16`, following `c_char`,
+  so no cast is needed at call sites; see the doc comment on `cxx::c_char16` for the limits that
+  follow from being an alias.
+- **Unqualified emission of C++ fundamental types.** A `Pair` whose name is a fundamental type and
+  whose namespace is empty is written bare rather than with a leading `::`, because those names are
+  keywords: `::char16_t` is ill-formed where `::uint16_t` is fine. This lets an `ExternType` alias
+  bind a fundamental type that has no builtin spelling in a bridge.
 - **Workerd build integration.** Bazel targets use workerd's in-tree Rust toolchain, crate vendor
   repository, Cap'n Proto/KJ dependency, formatting, and lint configuration.

src/rust/cxx/gen/src/write.rs

@@ -242,7 +242,9 @@ fn pick_includes_and_builtins(out: &mut OutFile, apis: &[Api]) {
                 Some(Isize) => out.builtin.rust_isize = true,
                 Some(CxxString) => out.include.string = true,
                 Some(RustString) => out.builtin.rust_string = true,
-                Some(Bool | Char | F32 | F64) | None => {}
+                // bool, char, char16_t, float and double are C++ keywords; no
+                // header supplies them.
+                Some(Bool | Char | Char16 | F32 | F64) | None => {}
             },
             Type::RustBox(_) => out.builtin.rust_box = true,
             Type::RustVec(_) => out.builtin.rust_vec = true,
@@ -1374,6 +1376,7 @@ fn write_atom(out: &mut OutFile, atom: Atom) {
     match atom {
         Bool => write!(out, "bool"),
         Char => write!(out, "char"),
+        Char16 => write!(out, "char16_t"),
         U8 => write!(out, "::std::uint8_t"),
         U16 => write!(out, "::std::uint16_t"),
         U32 => write!(out, "::std::uint32_t"),

src/rust/cxx/src/cxx.cc

@@ -437,6 +437,14 @@ static_assert(sizeof(rust::isize) == sizeof(std::intptr_t),
 static_assert(alignof(rust::isize) == alignof(std::intptr_t),
               "unsupported ssize_t alignment");
 
+// The cxx crate spells char16_t as `c_char16`, an alias for u16. char16_t's
+// underlying type is uint_least16_t, which the standard permits to be wider
+// than 16 bits; on such a target the alias would misdescribe the C++ type.
+static_assert(sizeof(char16_t) == sizeof(std::uint16_t),
+              "unsupported char16_t size");
+static_assert(alignof(char16_t) == alignof(std::uint16_t),
+              "unsupported char16_t alignment");
+
 static_assert(std::is_trivially_copy_constructible<Str>::value,
               "trivial Str(const Str &)");
 static_assert(std::is_trivially_copy_assignable<Str>::value,
@@ -827,10 +835,14 @@ static_assert(sizeof(std::string) <= kMaxExpectedWordsInString * sizeof(void *),
   FOR_EACH_TRIVIAL_STD_VECTOR(MACRO)                                           \
   MACRO(string, std::string)
 
+// char16_t needs no _if_unique guard: [basic.fundamental] makes it a distinct
+// type from every other fundamental type, so rust::Vec<char16_t> can never
+// collide with another specialization.
 #define FOR_EACH_RUST_VEC(MACRO)                                               \
   FOR_EACH_NUMERIC(MACRO)                                                      \
   MACRO(bool, bool)                                                            \
   MACRO(char, rust::detail::char_if_unique)                                    \
+  MACRO(char16_t, char16_t)                                                    \
   MACRO(usize, rust::detail::usize_if_unique)                                  \
   MACRO(isize, rust::detail::isize_if_unique)                                  \
   MACRO(string, rust::String)                                                  \

src/rust/cxx/src/lib.rs

@@ -467,6 +467,26 @@ pub type String = CxxString;
 /// import and use `CxxVector`.
 pub type Vector<T> = CxxVector<T>;
 
+/// Rust spelling of C++'s `char16_t`, for UTF-16 data crossing the bridge.
+///
+/// `char16_t` and `uint16_t` are layout-identical but distinct C++ types, so
+/// C++ mangles them differently and overloads on them separately. `u16` in a
+/// bridge means `uint16_t`; use `c_char16` where the signature must name
+/// `char16_t`, which is the usual case for UTF-16 buffers.
+///
+/// This is a plain alias rather than a newtype, following `c_char`, so a
+/// `&[c_char16]` argument accepts a `&[u16]` from `str::encode_utf16` or
+/// `encoding_rs` with no cast. The alias means `c_char16` and `u16` are one
+/// Rust type; only their C++ spelling differs. The two therefore cannot both
+/// be used as the element type of a `CxxVector` in the same program, and
+/// `CxxVector<c_char16>` is rejected for the same reason `CxxVector<c_char>`
+/// is.
+///
+/// `char16_t` is guaranteed by the C++ standard to be a distinct type whose
+/// underlying type is `uint_least16_t`. `cxx.cc` asserts that it really is 16
+/// bits wide on the target, which is what makes this alias sound.
+pub type c_char16 = u16;
+
 // Not public API.
 #[doc(hidden)]
 pub mod private {

src/rust/cxx/src/symbols/rust_vec.rs

@@ -3,6 +3,7 @@ use core::ffi::c_char;
 use core::mem;
 use core::ptr;
 
+use crate::c_char16;
 use crate::rust_string::RustString;
 use crate::rust_vec::RustVec;
 
@@ -78,5 +79,9 @@ rust_vec_shims_for_primitive!(f32);
 rust_vec_shims_for_primitive!(f64);
 
 rust_vec_shims!("char", c_char);
+// c_char16 is an alias for u16, so these duplicate the u16 shims above under a
+// second set of export names. The C++ side reaches them through
+// rust::Vec<char16_t>, which mangles to the char16_t segment.
+rust_vec_shims!("char16_t", c_char16);
 rust_vec_shims!("string", RustString);
 rust_vec_shims!("str", &str);

src/rust/cxx/syntax/atom.rs

@@ -9,6 +9,10 @@ use crate::Type;
 pub enum Atom {
     Bool,
     Char, // C char, not Rust char
+    // C++ char16_t. Layout-identical to u16 but a distinct C++ type, so it is a
+    // distinct atom: it exists to make overload resolution and name mangling on
+    // the C++ side pick char16_t rather than uint16_t.
+    Char16,
     U8,
     U16,
     U32,
@@ -35,6 +39,7 @@ impl Atom {
         match s {
             "bool" => Some(Bool),
             "c_char" => Some(Char),
+            "c_char16" => Some(Char16),
             "u8" => Some(U8),
             "u16" => Some(U16),
             "u32" => Some(U32),
@@ -66,6 +71,7 @@ impl AsRef<str> for Atom {
         match self {
             Bool => "bool",
             Char => "c_char",
+            Char16 => "c_char16",
             U8 => "u8",
             U16 => "u16",
             U32 => "u32",

src/rust/cxx/syntax/check.rs

@@ -163,8 +163,8 @@ fn check_type_rust_vec(cx: &mut Check, ty: &Ty1) {
             match Atom::from(&ident.rust) {
                 None
                 | Some(
-                    Bool | Char | U8 | U16 | U32 | U64 | Usize | I8 | I16 | I32 | I64 | Isize | F32
-                    | F64 | RustString,
+                    Bool | Char | Char16 | U8 | U16 | U32 | U64 | Usize | I8 | I16 | I32 | I64
+                    | Isize | F32 | F64 | RustString,
                 ) => return,
                 Some(CxxString) => {}
             }
@@ -266,7 +266,7 @@ fn check_type_shared_ptr(cx: &mut Check, ptr: &Ty1) {
                 Bool | U8 | U16 | U32 | U64 | Usize | I8 | I16 | I32 | I64 | Isize | F32 | F64
                 | CxxString,
             ) => return,
-            Some(Char | RustString) => {}
+            Some(Char | Char16 | RustString) => {}
         }
     } else if let Type::CxxVector(_) = &ptr.inner {
         cx.error(ptr, "std::shared_ptr<std::vector> is not supported yet");
@@ -289,7 +289,7 @@ fn check_type_weak_ptr(cx: &mut Check, ptr: &Ty1) {
                 Bool | U8 | U16 | U32 | U64 | Usize | I8 | I16 | I32 | I64 | Isize | F32 | F64
                 | CxxString,
             ) => return,
-            Some(Char | RustString) => {}
+            Some(Char | Char16 | RustString) => {}
         }
     } else if let Type::CxxVector(_) = &ptr.inner {
         cx.error(ptr, "std::weak_ptr<std::vector> is not supported yet");
@@ -363,7 +363,7 @@ fn check_type_cxx_vector(cx: &mut Check, ptr: &Ty1) {
             | Some(
                 U8 | U16 | U32 | U64 | Usize | I8 | I16 | I32 | I64 | Isize | F32 | F64 | CxxString,
             ) => return,
-            Some(Char) => { /* todo */ }
+            Some(Char | Char16) => { /* todo */ }
             Some(Bool | RustString) => {}
         }
     }
@@ -900,6 +900,8 @@ fn describe(cx: &mut Check, ty: &Type) -> String {
                 "C++ string".to_owned()
             } else if Atom::from(&ident.rust) == Some(Char) {
                 "C char".to_owned()
+            } else if Atom::from(&ident.rust) == Some(Char16) {
+                "C++ char16_t".to_owned()
             } else {
                 ident.rust.to_string()
             }

src/rust/cxx/syntax/names.rs

@@ -32,6 +32,15 @@ impl Pair {
     }
 
     pub fn to_fully_qualified(&self) -> String {
+        // A fundamental type's name is a keyword, not something declared in a
+        // scope, so it is spelled bare: `::char16_t` is ill-formed where
+        // `::uint16_t` is fine. Reaching this with a non-empty namespace means
+        // the name was not really a fundamental type, so leave it qualified
+        // and let C++ report it.
+        if self.namespace.is_empty() && self.cxx.is_fundamental() {
+            return self.cxx.to_string();
+        }
+
         let mut fully_qualified = String::new();
         for segment in &self.namespace {
             fully_qualified += "::";
@@ -72,6 +81,35 @@ impl ForeignName {
             Err(err) => Err(Error::new(span, err)),
         }
     }
+
+    /// True if this names a C++ fundamental type.
+    ///
+    /// These are the only C++ type names that are keywords, and so the only
+    /// ones that cannot be written with a `::` qualification. Fixed-width
+    /// names like `uint16_t` are ordinary typedefs, not keywords, and are
+    /// deliberately absent.
+    ///
+    /// Multi-token spellings such as `unsigned int` are absent because
+    /// [`ForeignName::parse`] cannot represent a name containing whitespace.
+    pub fn is_fundamental(&self) -> bool {
+        matches!(
+            self.text.as_str(),
+            "bool"
+                | "char"
+                | "char8_t"
+                | "char16_t"
+                | "char32_t"
+                | "double"
+                | "float"
+                | "int"
+                | "long"
+                | "short"
+                | "signed"
+                | "unsigned"
+                | "void"
+                | "wchar_t"
+        )
+    }
 }
 
 impl Display for ForeignName {

src/rust/cxx/syntax/namespace.rs

@@ -33,6 +33,11 @@ impl Namespace {
         self.segments.iter()
     }
 
+    /// True for the global namespace.
+    pub fn is_empty(&self) -> bool {
+        self.segments.is_empty()
+    }
+
     pub fn parse_bridge_attr_namespace(input: ParseStream) -> Result<Self> {
         if input.is_empty() {
             return Ok(Self::ROOT);

src/rust/cxx/syntax/pod.rs

@@ -12,8 +12,8 @@ impl Types<'_> {
                 let ident = &ident.rust;
                 if let Some(atom) = Atom::from(ident) {
                     match atom {
-                        Bool | Char | U8 | U16 | U32 | U64 | Usize | I8 | I16 | I32 | I64
-                        | Isize | F32 | F64 => true,
+                        Bool | Char | Char16 | U8 | U16 | U32 | U64 | Usize | I8 | I16 | I32
+                        | I64 | Isize | F32 | F64 => true,
                         CxxString | RustString => false,
                     }
                 } else if let Some(strct) = self.structs.get(ident) {

src/rust/cxx/syntax/tokens.rs

@@ -36,6 +36,11 @@ impl ToTokens for Type {
                 if ident.rust == Char {
                     let span = ident.rust.span();
                     tokens.extend(quote_spanned!(span=> ::cxx::core::ffi::));
+                } else if ident.rust == Char16 {
+                    // Unlike c_char, there is no core::ffi equivalent of
+                    // char16_t; the cxx crate defines c_char16 itself.
+                    let span = ident.rust.span();
+                    tokens.extend(quote_spanned!(span=> ::cxx::));
                 } else if ident.rust == CxxString {
                     let span = ident.rust.span();
                     tokens.extend(quote_spanned!(span=> ::cxx::));

src/rust/cxx/tests/cxx_gen.rs

@@ -88,6 +88,33 @@ const BRIDGE4: &str = r#"
     }
 "#;
 
+const BRIDGE5: &str = r#"
+    #[cxx::bridge]
+    mod ffi {
+        unsafe extern "C++" {
+            fn utf16_length(s: &[c_char16]) -> usize;
+            fn widen(input: &[u8], output: &mut [c_char16]) -> usize;
+            unsafe fn first(s: *const c_char16) -> c_char16;
+        }
+
+        extern "Rust" {
+            fn encode(s: &str) -> Vec<c_char16>;
+        }
+    }
+"#;
+
+const BRIDGE6: &str = r#"
+    #[cxx::bridge]
+    mod ffi {
+        unsafe extern "C++" {
+            #[cxx_name = "char32_t"]
+            type Char32 = crate::Char32;
+
+            fn count(s: &[Char32]) -> usize;
+        }
+    }
+"#;
+
 #[test]
 fn test_extern_c_function() {
     let opt = cxx_gen::Opt::default();
@@ -177,3 +204,30 @@ fn test_kj_arc_in_shared_struct() {
     let expected = "::rust::ManuallyDrop<::Holder> holder$(::std::move(holder));";
     assert!(implementation.contains(expected));
 }
+
+#[test]
+fn test_c_char16_maps_to_cxx_char16_t() {
+    let opt = cxx_gen::Opt::default();
+    let source = BRIDGE5.parse().unwrap();
+    let generated = generate_header_and_cc(source, &opt).unwrap();
+    let implementation = str::from_utf8(&generated.implementation).unwrap();
+    assert!(implementation.contains("::rust::Slice<char16_t const>"));
+    assert!(implementation.contains("::rust::Slice<char16_t >"));
+    assert!(implementation.contains("char16_t const *"));
+    assert!(implementation.contains("::rust::Vec<char16_t>"));
+    // char16_t must not decay to uint16_t: keeping the two distinct is the
+    // whole reason c_char16 is a separate atom from u16.
+    assert!(!implementation.contains("uint16_t"));
+}
+
+#[test]
+fn test_fundamental_type_name_is_unqualified() {
+    let opt = cxx_gen::Opt::default();
+    let source = BRIDGE6.parse().unwrap();
+    let generated = generate_header_and_cc(source, &opt).unwrap();
+    let implementation = str::from_utf8(&generated.implementation).unwrap();
+    assert!(implementation.contains("::rust::Slice<char32_t const>"));
+    // `::char32_t` is ill-formed; a fundamental type's name is a keyword and
+    // cannot be qualified.
+    assert!(!implementation.contains("::char32_t"));
+}

src/rust/cxx/tests/ffi/lib.rs

@@ -29,6 +29,7 @@ use cxx::KjError;
 use cxx::KjExceptionType;
 use cxx::SharedPtr;
 use cxx::UniquePtr;
+use cxx::c_char16;
 use cxx::type_id;
 // The bridge parser accepts the unqualified smart-pointer name, while expansion
 // fully qualifies the emitted Rust field type.
@@ -153,6 +154,8 @@ pub mod ffi {
         unsafe fn c_return_mut<'a>(shared: &'a mut Shared) -> &'a mut usize;
         unsafe fn c_return_str<'a>(shared: &'a Shared) -> &'a str;
         unsafe fn c_return_slice_char<'a>(shared: &'a Shared) -> &'a [c_char];
+        unsafe fn c_return_slice_char16<'a>(shared: &'a Shared) -> &'a [c_char16];
+        fn c_return_rust_vec_char16() -> Vec<c_char16>;
         unsafe fn c_return_mutsliceu8<'a>(slice: &'a mut [u8]) -> &'a mut [u8];
         unsafe fn c_return_ref<'a>(shared: &'a Shared) -> &'a usize;
         fn c_return_rust_string() -> String;
@@ -245,6 +248,8 @@ pub mod ffi {
         fn c_take_ref_c(c: &C);
         fn c_take_str(s: &str);
         fn c_take_slice_char(s: &[c_char]);
+        fn c_take_slice_char16(s: &[c_char16]);
+        fn c_take_rust_vec_char16(v: Vec<c_char16>);
         fn c_take_slice_shared(s: &[Shared]);
         fn c_take_slice_shared_sort(s: &mut [Shared]);
         fn c_take_slice_r(s: &[R]);
@@ -430,6 +435,8 @@ pub mod ffi {
         fn r_take_ref_c(c: &C);
         fn r_take_str(s: &str);
         fn r_take_slice_char(s: &[c_char]);
+        fn r_take_slice_char16(s: &[c_char16]);
+        fn r_return_rust_vec_char16() -> Vec<c_char16>;
         fn r_take_rust_string(s: String);
         fn r_take_unique_ptr_string(s: UniquePtr<CxxString>);
         fn r_take_ref_vector(v: &CxxVector<u8>);
@@ -758,6 +765,16 @@ fn r_take_slice_char(s: &[c_char]) {
     assert_eq!(std::str::from_utf8(s), Ok("2020\0"));
 }
 
+// c_char16 is an alias for u16, so a &[c_char16] is a &[u16] and UTF-16 from
+// the standard library needs no conversion to cross the bridge.
+fn r_take_slice_char16(s: &[c_char16]) {
+    assert_eq!(String::from_utf16_lossy(s), "2020");
+}
+
+fn r_return_rust_vec_char16() -> Vec<c_char16> {
+    "2020".encode_utf16().collect()
+}
+
 fn r_take_unique_ptr_string(s: UniquePtr<CxxString>) {
     assert_eq!(
         s.as_ref().and_then(|value| value.to_str().ok()),