Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions docs/jsg.md
Original file line number Diff line number Diff line change
Expand Up @@ -1841,6 +1841,16 @@ v8::Local<v8::String> newExternalOneByteString(Lock& js, kj::ArrayPtr<const char
v8::Local<v8::String> newExternalTwoByteString(Lock& js, kj::ArrayPtr<const uint16_t> buf);
```

For dynamically allocated buffers, use the ownership-carrying overloads:

```cpp
using OwnedAscii = kj::Array<const char>;
using OwnedUtf16 = kj::Array<const uint16_t>;

v8::Local<v8::String> newExternalOneByteString(Lock& js, kj::Arc<OwnedAscii> buf);
v8::Local<v8::String> newExternalTwoByteString(Lock& js, kj::Arc<OwnedUtf16> buf);
```

**Important:** The `OneByteString` variant interprets the buffer as Latin-1, not UTF-8.

---
Expand Down Expand Up @@ -2484,7 +2494,7 @@ bundles, import.meta support, import attributes.

```cpp
auto esmModule = Module::newEsm("file:///bundle/worker.js"_url,
Module::Type::BUNDLE, kj::mv(code),
Module::Type::BUNDLE, kj::arc<OwnedAscii>(kj::mv(code)),
Module::Flags::MAIN | Module::Flags::ESM);

auto syntheticModule = Module::newSynthetic("workerd:my-module"_url,
Expand Down Expand Up @@ -2514,14 +2524,16 @@ auto wasmHandler = Module::newWasmModuleHandler(wasmBytes);
Url bundleBase = "file:///bundle"_url;
ModuleBundle::BundleBuilder bundleBuilder(bundleBase);
bundleBuilder
.addEsmModule("worker.js", workerSource, Module::Flags::MAIN | Module::Flags::ESM)
.addEsmModule("utils.js", utilsSource)
.addEsmModule("worker.js", kj::arc<OwnedAscii>(kj::mv(workerSource)),
Module::Flags::MAIN | Module::Flags::ESM)
.addEsmModule("utils.js", kj::arc<OwnedAscii>(kj::mv(utilsSource)))
.alias("./lib", "./utils.js");
auto workerBundle = bundleBuilder.finish();

// Builtin bundle
ModuleBundle::BuiltinBuilder builtinBuilder(ModuleBundle::BuiltinBuilder::Type::BUILTIN);
builtinBuilder
// Builtin ESM source must have static process lifetime.
.addEsm("node:buffer"_url, bufferSource)
.addSynthetic("cloudflare:sockets"_url, socketsHandler)
.addObject<MyApiClass, MyTypeWrapper>("workerd:my-api"_url);
Expand All @@ -2543,7 +2555,7 @@ auto fallbackBundle = ModuleBundle::newFallbackBundle(
-> kj::Maybe<kj::OneOf<kj::String, kj::Own<Module>>> {
KJ_IF_SOME(code, fetchModule(context.normalizedSpecifier)) {
return Module::newEsm(context.normalizedSpecifier.clone(),
Module::Type::FALLBACK, kj::mv(code));
Module::Type::FALLBACK, kj::arc<OwnedAscii>(kj::mv(code)));
}
if (shouldRedirect(context.normalizedSpecifier)) {
return kj::str("node:buffer"); // Redirect
Expand Down
19 changes: 12 additions & 7 deletions docs/reference/detail/new-module-registry.md
Original file line number Diff line number Diff line change
Expand Up @@ -160,9 +160,9 @@ module construction time.

```
EsModule extends Module {
source: kj::ArrayPtr<const char> // Raw UTF-8 source text
encodedSource: kj::Lazy<EncodedSource> // V8-compatible encoding (shared)
cachedData: MutexGuarded<Maybe<Own<CachedData>>> // Cross-isolate compile cache
source: kj::OneOf<kj::ArrayPtr<const char>, kj::Arc<OwnedAscii>> // UTF-8
encodedSource: kj::Lazy<EncodedSource> // V8-compatible encoding (shared)
cachedData: MutexGuarded<Maybe<Own<CachedData>>> // Cross-isolate compile cache
}
```

Expand All @@ -181,9 +181,9 @@ isolate that compiles the module:

| Source content | Representation |
| ---------------------------------- | ------------------------------------------------ |
| Pure ASCII (the common case) | Zero-copy one-byte external over the raw buffer |
| Non-ASCII, all code points ≤ U+00FF | One-time UTF-8→Latin-1 transcode (one-byte) |
| Anything else (CJK, emoji, ...) | One-time UTF-8→UTF-16 transcode (two-byte) |
| Pure ASCII (the common case) | Zero-copy one-byte external over static or Arc-owned storage |
| Non-ASCII, all code points ≤ U+00FF | One-time UTF-8→Latin-1 transcode into Arc-owned storage |
| Anything else (CJK, emoji, ...) | One-time UTF-8→UTF-16 transcode into Arc-owned storage |

Invalid UTF-8 sequences are replaced with U+FFFD (via `kj::encodeUtf16`),
matching `v8::String::NewFromUtf8`'s tolerance and the legacy registry. The
Expand Down Expand Up @@ -274,12 +274,17 @@ to the `bundleBase` URL (typically `file:///bundle/`):

```cpp
BundleBuilder builder(bundleBase);
builder.addEsmModule("index.js", source, Flags::ESM | Flags::MAIN);
builder.addEsmModule(
"index.js", kj::arc<OwnedAscii>(kj::mv(source)), Flags::ESM | Flags::MAIN);
builder.addSyntheticModule("data.json", Module::newJsonModuleHandler(jsonData));
builder.addWasmModule("module.wasm", wasmBytes);
auto bundle = builder.finish();
```

The `kj::Arc<OwnedAscii>` overload must be used for ordinary worker source. The
`ArrayPtr` overload is reserved for compiled-in strings with static process
lifetime because V8 can retain an external source string after compilation.

Name normalization (`normalizeModuleName`):

1. Parse name as URL relative to `bundleBase`.
Expand Down
20 changes: 6 additions & 14 deletions src/workerd/io/worker-modules.h
Original file line number Diff line number Diff line change
Expand Up @@ -141,20 +141,12 @@ static kj::Arc<jsg::modules::ModuleRegistry> newWorkerModuleRegistry(
if (def.name == source.mainModule) {
flags = flags | jsg::modules::Module::Flags::MAIN;
}
if (content.ownBody != kj::none) {
// When the source is owned (e.g. transpiled TypeScript), we must
// copy it into the module registry since the owning rust::String
// may not outlive the registry.
bundleBuilder.addEsmModule(def.name, kj::heapArray<const char>(content.body), flags);
} else {
// The content.body points into memory that outlives the module
// registry. In workerd this is a process-lifetime capnp message
// buffer; in edgeworker, it is the disowned script-fetcher response
// owned by the VirtualFileSystem (which is a sibling of the registry
// in Worker::Script::Impl). In edgeworker, the copy is ensured by
// shouldCopyScriptFetcherResponse() including the NMR flag.
bundleBuilder.addEsmModule(def.name, content.body, flags);
}
// Worker bundle storage is not necessarily static: transpiled source
// can be backed by a temporary rust::String, and edgeworker source is
// backed by a script-fetcher response. Copy it once into shared storage
// so V8 external strings can safely outlive the registry.
bundleBuilder.addEsmModule(
def.name, kj::arc<jsg::OwnedAscii>(kj::heapArray<const char>(content.body)), flags);
break;
}
KJ_CASE_ONEOF(content, Worker::Script::TextModule) {
Expand Down
2 changes: 2 additions & 0 deletions src/workerd/jsg/jsg.h
Original file line number Diff line number Diff line change
Expand Up @@ -3151,7 +3151,9 @@ class Lock {
JsString str(kj::ArrayPtr<const kj::byte>) KJ_WARN_UNUSED_RESULT;
JsString strIntern(kj::StringPtr) KJ_WARN_UNUSED_RESULT;
JsString strExtern(kj::ArrayPtr<const char>) KJ_WARN_UNUSED_RESULT;
JsString strExtern(kj::Arc<OwnedAscii>) KJ_WARN_UNUSED_RESULT;
JsString strExtern(kj::ArrayPtr<const uint16_t>) KJ_WARN_UNUSED_RESULT;
JsString strExtern(kj::Arc<OwnedUtf16>) KJ_WARN_UNUSED_RESULT;
JsSymbol symbol(kj::StringPtr) KJ_WARN_UNUSED_RESULT;
JsSymbol symbolShared(kj::StringPtr) KJ_WARN_UNUSED_RESULT;
JsSymbol symbolInternal(kj::StringPtr) KJ_WARN_UNUSED_RESULT;
Expand Down
8 changes: 8 additions & 0 deletions src/workerd/jsg/jsvalue.h
Original file line number Diff line number Diff line change
Expand Up @@ -1370,10 +1370,18 @@ inline JsString Lock::strExtern(kj::ArrayPtr<const char> str) {
return JsString(newExternalOneByteString(*this, str));
}

inline JsString Lock::strExtern(kj::Arc<OwnedAscii> str) {
return JsString(newExternalOneByteString(*this, kj::mv(str)));
}

inline JsString Lock::strExtern(kj::ArrayPtr<const uint16_t> str) {
return JsString(newExternalTwoByteString(*this, str));
}

inline JsString Lock::strExtern(kj::Arc<OwnedUtf16> str) {
return JsString(newExternalTwoByteString(*this, kj::mv(str)));
}

inline JsObject Lock::obj() {
return JsObject(v8::Object::New(v8Isolate));
}
Expand Down
71 changes: 59 additions & 12 deletions src/workerd/jsg/modules-new-test.c++
Original file line number Diff line number Diff line change
Expand Up @@ -1608,21 +1608,22 @@ KJ_TEST("Module source is decoded as UTF-8 across all encoding tiers") {
// ======================================================================================

KJ_TEST("Owned ESM source outlives its release points across encoding tiers") {
// The kj::Array-taking addEsmModule overload hands ownership of the UTF-8
// source buffer to the module. Non-ASCII sources are transcoded to an owned
// The Arc<OwnedAscii> addEsmModule overload shares ownership of the UTF-8 source
// buffer with the module. Non-ASCII sources are transcoded to an owned
// V8-compatible representation on first compile, after which the UTF-8
// original is released; pure-ASCII owned sources must be retained because
// the raw buffer directly backs the external one-byte string. This test
// asserts correctness across repeated resolution; a buffer released too
// early (or read after release) is observed by ASAN builds.
// original is released; pure-ASCII owned sources become the shared encoded
// representation. V8 external strings retain shared ownership independently
// of the module. This test asserts correctness across repeated resolution; a
// buffer released too early (or read after release) is observed by ASAN builds.
PREAMBLE([&](Lock& js) {
CompilationObserver compilationObserver;

ModuleBundle::BundleBuilder bundleBuilder(BASE);
bundleBuilder.addEsmModule(
"latin1-owned", kj::heapArray<const char>("export default 'caf\xc3\xa9';"_kj.asArray()));
bundleBuilder.addEsmModule(
"ascii-owned", kj::heapArray<const char>("export default 'plain';"_kj.asArray()));
bundleBuilder.addEsmModule("latin1-owned",
kj::arc<OwnedAscii>(
kj::heapArray<const char>("export default 'caf\xc3\xa9';"_kj.asArray())));
bundleBuilder.addEsmModule("ascii-owned",
kj::arc<OwnedAscii>(kj::heapArray<const char>("export default 'plain';"_kj.asArray())));

auto registry = ModuleRegistry::Builder(BASE).add(bundleBuilder.finish()).finish();
auto attached = registry->attachToIsolate(js, compilationObserver);
Expand All @@ -1646,6 +1647,51 @@ KJ_TEST("Owned ESM source outlives its release points across encoding tiers") {

// ======================================================================================

KJ_TEST("Compiled ESM functions outlive owned source across encoding tiers") {
PREAMBLE([&](Lock& js) {
CompilationObserver compilationObserver;
kj::Vector<JsRef<JsFunction>> functions;

JSG_TRY(js) {
{
ModuleBundle::BundleBuilder bundleBuilder(BASE);
bundleBuilder.addEsmModule("ascii-lifetime",
kj::arc<OwnedAscii>(kj::heapArray<const char>(
"export default function deferred() { return 'plain'; }"_kj.asArray())));
bundleBuilder.addEsmModule("latin1-lifetime",
kj::arc<OwnedAscii>(kj::heapArray<const char>(
"export default function deferred() { return 'caf\xc3\xa9'; }"_kj.asArray())));
bundleBuilder.addEsmModule("utf16-lifetime",
kj::arc<OwnedAscii>(kj::heapArray<const char>(
"export default function deferred() { return '\xe9\x83\xa8\xe5\x93\x81 \xf0\x9f\x8e\x89'; }"_kj
.asArray())));

auto registry = ModuleRegistry::Builder(BASE).add(bundleBuilder.finish()).finish();
auto attached = registry->attachToIsolate(js, compilationObserver);

for (auto specifier: {"file:///ascii-lifetime"_kj, "file:///latin1-lifetime"_kj,
"file:///utf16-lifetime"_kj}) {
auto value = ModuleRegistry::resolve(js, specifier);
auto function = KJ_ASSERT_NONNULL(value.tryCast<JsFunction>());
functions.add(JsRef<JsFunction>(js, function));
}
}

// The exported functions remain live in V8 after the registry, bundles,
// and their owned source buffers have been destroyed.
KJ_ASSERT(kj::str(functions[0].getHandle(js).call(js, js.null())) == "plain");
KJ_ASSERT(kj::str(functions[1].getHandle(js).call(js, js.null())) == "caf\xc3\xa9");
KJ_ASSERT(kj::str(functions[2].getHandle(js).call(js, js.null())) ==
"\xe9\x83\xa8\xe5\x93\x81 \xf0\x9f\x8e\x89");
}
JSG_CATCH(exception) {
js.throwException(kj::mv(exception));
}
});
}

// ======================================================================================

KJ_TEST("Dynamic import from within a CJS-style eval module works") {
PREAMBLE([&](Lock& js) {
ResolveObserverImpl observer;
Expand Down Expand Up @@ -2161,8 +2207,9 @@ KJ_TEST("UNWRAP_DEFAULT honors module.exports, marker order, and builtin fallbac
[](const ResolveContext& context) -> kj::Maybe<kj::OneOf<kj::String, kj::Own<Module>>> {
auto source = kj::heapArray<const char>(
"export default 'fb-default'; export const named = 'fb';"_kj.asArray());
return kj::Maybe<kj::OneOf<kj::String, kj::Own<Module>>>(Module::newEsm(
context.normalizedSpecifier.clone(), Module::Type::FALLBACK, kj::mv(source)));
return kj::Maybe<kj::OneOf<kj::String, kj::Own<Module>>>(
Module::newEsm(context.normalizedSpecifier.clone(), Module::Type::FALLBACK,
kj::arc<OwnedAscii>(kj::mv(source))));
});

auto registry = ModuleRegistry::Builder(BASE, ModuleRegistry::Builder::Options::ALLOW_FALLBACK)
Expand Down
Loading
Loading