Skip to content

Commit 03207b6

Browse files
seyeong-hanseyeong-han
committed
Add persistent Supertonic JSONL mode
1 parent 9377ded commit 03207b6

4 files changed

Lines changed: 542 additions & 4 deletions

File tree

‎examples/models/supertonic/CMakeLists.txt‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,3 +141,18 @@ set_tests_properties(
141141
PROPERTIES TIMEOUT 1800 SKIP_REGULAR_EXPRESSION
142142
"SKIP: Supertonic integration assets are unavailable"
143143
)
144+
add_test(
145+
NAME supertonic_jsonl_integration
146+
COMMAND
147+
${CMAKE_COMMAND} -DRUNNER=$<TARGET_FILE:supertonic_runner>
148+
-DPTE=${SUPERTONIC_INTEGRATION_PTE}
149+
-DASSET_DIR=${SUPERTONIC_INTEGRATION_ASSET_DIR}
150+
-DSTYLE=${SUPERTONIC_INTEGRATION_STYLE}
151+
-DWORK_DIR=${CMAKE_CURRENT_BINARY_DIR}/jsonl-integration -P
152+
${CMAKE_CURRENT_SOURCE_DIR}/runtime/tests/run_jsonl_server_integration.cmake
153+
)
154+
set_tests_properties(
155+
supertonic_jsonl_integration
156+
PROPERTIES TIMEOUT 1800 SKIP_REGULAR_EXPRESSION
157+
"SKIP: Supertonic JSONL server assets are unavailable"
158+
)

‎examples/models/supertonic/README.md‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# Supertonic on ExecuTorch MLX
22

33
This example exports [Supertonic 3](https://huggingface.co/Supertone/supertonic-3)
4-
to one dynamic FP16 ExecuTorch program and performs one-shot text-to-speech
5-
synthesis with the MLX delegate.
4+
to one dynamic FP16 ExecuTorch program and performs one-shot or persistent
5+
text-to-speech synthesis with the MLX delegate.
66

77
Run all commands from the ExecuTorch repository root. Keep downloaded assets
88
and generated programs outside the source tree:
@@ -70,6 +70,43 @@ The runner is written to
7070
The runner writes a mono PCM16 WAV at the sample rate recorded in the model
7171
metadata (44.1 kHz for the pinned assets).
7272

73+
## 5. Keep the runner warm
74+
75+
Use `--server_jsonl` to load the PTE once, perform one discarded warmup, and
76+
process synthesis requests sequentially from stdin:
77+
78+
```bash
79+
./cmake-out/examples/models/supertonic/supertonic_runner \
80+
--pte="${SUPERTONIC_PTE}" \
81+
--asset_dir="${SUPERTONIC_ASSETS}" \
82+
--voice_style="${SUPERTONIC_ASSETS}/voice_styles/F1.json" \
83+
--language=en \
84+
--speed=1.05 \
85+
--seed=42 \
86+
--server_jsonl
87+
```
88+
89+
After loading and warmup, the runner emits this protocol-v1 `ready` schema;
90+
`sample_rate` is always `44100` for the supported model:
91+
92+
```json
93+
{"type":"ready","protocol_version":1,"sample_rate":44100,"load_seconds":0.03,"warmup_seconds":0.04}
94+
```
95+
96+
Each request writes one complete WAV before returning its timing and RTF:
97+
98+
```json
99+
{"type":"synthesize","id":1,"text":"Hello.","output":"/tmp/supertonic/1.wav"}
100+
{"type":"result","id":1,"output":"/tmp/supertonic/1.wav","samples":82810,"audio_seconds":1.8778,"synthesis_seconds":0.0524,"rtf":0.0279}
101+
```
102+
103+
Stdout is reserved for JSONL responses and flushed after every record;
104+
diagnostics go to stderr. Request lines are limited to 64 KiB, and IDs must be
105+
positive and strictly increasing. Output files are created atomically and must
106+
not already exist. Malformed requests return an `error` record without stopping
107+
the runner. Send `{"type":"shutdown"}` to receive `{"type":"stopped"}` and
108+
exit cleanly; closing stdin also exits with status zero.
109+
73110
## Platform and model limits
74111

75112
- This workflow requires an Apple silicon Mac, macOS, Xcode command-line

‎examples/models/supertonic/runtime/main.cpp‎

Lines changed: 259 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,19 @@
1010
#include "wav_writer.h"
1111

1212
#include <gflags/gflags.h>
13+
#include <nlohmann/json.hpp>
1314

15+
#include <unistd.h>
16+
#include <cerrno>
17+
#include <chrono>
1418
#include <cmath>
19+
#include <cstdio>
20+
#include <cstring>
1521
#include <filesystem>
1622
#include <iostream>
1723
#include <limits>
24+
#include <optional>
25+
#include <set>
1826
#include <stdexcept>
1927
#include <string>
2028

@@ -32,16 +40,254 @@ DEFINE_string(language, "en", "Published Supertonic language tag.");
3240
DEFINE_double(speed, 1.05, "Positive duration speed divisor.");
3341
DEFINE_uint64(seed, 42, "Portable xorshift64/Box-Muller latent seed.");
3442
DEFINE_string(output, "supertonic.wav", "Output mono PCM16 WAV path.");
43+
DEFINE_bool(
44+
server_jsonl,
45+
false,
46+
"Keep the model resident and serve JSONL requests on stdin.");
47+
DEFINE_string(
48+
warmup_text,
49+
"Warmup.",
50+
"Text synthesized and discarded before the JSONL ready event.");
3551

3652
namespace {
3753

54+
using Json = nlohmann::ordered_json;
55+
56+
constexpr size_t kMaxJsonLineBytes = 64 * 1024;
57+
3858
void require_file(const std::filesystem::path& path, const char* flag) {
3959
if (!std::filesystem::is_regular_file(path)) {
4060
throw std::invalid_argument(
4161
std::string(flag) + " does not name a readable file: " + path.string());
4262
}
4363
}
4464

65+
void write_json_line(const Json& value) {
66+
std::cout << value.dump() << '\n';
67+
std::cout.flush();
68+
if (!std::cout) {
69+
throw std::runtime_error("failed to write JSONL response");
70+
}
71+
}
72+
73+
void write_error(std::optional<uint64_t> id, const std::string& message) {
74+
write_json_line({
75+
{"type", "error"},
76+
{"id", id.has_value() ? Json(*id) : Json(nullptr)},
77+
{"message", message},
78+
});
79+
}
80+
81+
double elapsed_seconds(std::chrono::steady_clock::time_point started) {
82+
const double elapsed =
83+
std::chrono::duration<double>(std::chrono::steady_clock::now() - started)
84+
.count();
85+
if (!std::isfinite(elapsed) || elapsed < 0.0) {
86+
throw std::runtime_error("elapsed time is not finite and nonnegative");
87+
}
88+
return elapsed;
89+
}
90+
91+
std::optional<std::string> validate_output_path(
92+
const std::filesystem::path& output) {
93+
std::error_code error;
94+
if (std::filesystem::exists(output, error)) {
95+
return "output already exists: " + output.string();
96+
}
97+
if (error) {
98+
return "failed to inspect output path: " + output.string();
99+
}
100+
const std::filesystem::path parent = output.parent_path().empty()
101+
? std::filesystem::path(".")
102+
: output.parent_path();
103+
if (!std::filesystem::is_directory(parent, error)) {
104+
return "output parent is not an existing directory: " + parent.string();
105+
}
106+
if (error) {
107+
return "failed to inspect output parent: " + parent.string();
108+
}
109+
return std::nullopt;
110+
}
111+
112+
enum class LineReadResult {
113+
Line,
114+
End,
115+
TooLong,
116+
};
117+
118+
LineReadResult read_json_line(std::istream& input, std::string& line) {
119+
line.clear();
120+
char character;
121+
while (input.get(character)) {
122+
if (character == '\n') {
123+
return LineReadResult::Line;
124+
}
125+
if (line.size() == kMaxJsonLineBytes) {
126+
while (input.get(character) && character != '\n') {
127+
}
128+
return LineReadResult::TooLong;
129+
}
130+
line.push_back(character);
131+
}
132+
return line.empty() ? LineReadResult::End : LineReadResult::Line;
133+
}
134+
135+
void write_wav_without_overwrite(
136+
const std::filesystem::path& output,
137+
const std::vector<float>& waveform,
138+
int sample_rate) {
139+
const std::filesystem::path parent = output.parent_path().empty()
140+
? std::filesystem::path(".")
141+
: output.parent_path();
142+
std::string temporary_template =
143+
(parent / ("." + output.filename().string() + ".XXXXXX")).string();
144+
int temporary_fd = mkstemp(temporary_template.data());
145+
if (temporary_fd < 0) {
146+
throw std::runtime_error(
147+
"failed to create temporary WAV: " + output.string() + ": " +
148+
std::strerror(errno));
149+
}
150+
const std::filesystem::path temporary(temporary_template);
151+
std::error_code cleanup_error;
152+
try {
153+
const std::string fd_path = "/dev/fd/" + std::to_string(temporary_fd);
154+
if (!supertonic::write_pcm16_wav(fd_path, waveform, sample_rate)) {
155+
throw std::runtime_error(
156+
"failed to write output WAV: " + output.string());
157+
}
158+
if (close(temporary_fd) != 0) {
159+
const int close_error = errno;
160+
temporary_fd = -1;
161+
throw std::runtime_error(
162+
"failed to close output WAV: " + output.string() + ": " +
163+
std::strerror(close_error));
164+
}
165+
temporary_fd = -1;
166+
if (renamex_np(temporary.c_str(), output.c_str(), RENAME_EXCL) != 0) {
167+
const int rename_error = errno;
168+
if (rename_error == EEXIST) {
169+
throw std::invalid_argument(
170+
"output already exists: " + output.string());
171+
}
172+
throw std::runtime_error(
173+
"failed to finalize output WAV: " + output.string() + ": " +
174+
std::strerror(rename_error));
175+
}
176+
} catch (...) {
177+
if (temporary_fd >= 0) {
178+
close(temporary_fd);
179+
}
180+
std::filesystem::remove(temporary, cleanup_error);
181+
throw;
182+
}
183+
}
184+
185+
int run_jsonl_server(
186+
supertonic::SupertonicRunner& runner,
187+
const supertonic::SynthesisOptions& defaults,
188+
double load_seconds) {
189+
supertonic::SynthesisOptions warmup = defaults;
190+
warmup.text = FLAGS_warmup_text;
191+
const auto warmup_started = std::chrono::steady_clock::now();
192+
(void)runner.synthesize(warmup);
193+
write_json_line({
194+
{"type", "ready"},
195+
{"protocol_version", 1},
196+
{"sample_rate", runner.metadata().sample_rate},
197+
{"load_seconds", load_seconds},
198+
{"warmup_seconds", elapsed_seconds(warmup_started)},
199+
});
200+
201+
uint64_t last_request_id = 0;
202+
std::string line;
203+
while (true) {
204+
const LineReadResult line_result = read_json_line(std::cin, line);
205+
if (line_result == LineReadResult::End) {
206+
return 0;
207+
}
208+
if (line_result == LineReadResult::TooLong) {
209+
write_error(std::nullopt, "JSON request exceeds 65536 bytes");
210+
continue;
211+
}
212+
std::optional<uint64_t> id;
213+
try {
214+
const Json request = Json::parse(line);
215+
if (!request.is_object() || !request.contains("type") ||
216+
!request["type"].is_string()) {
217+
throw std::invalid_argument("request must be an object with type");
218+
}
219+
const std::string type = request["type"].get<std::string>();
220+
if (type == "shutdown") {
221+
if (request.size() != 1) {
222+
throw std::invalid_argument("shutdown request has unexpected fields");
223+
}
224+
write_json_line({{"type", "stopped"}});
225+
return 0;
226+
}
227+
if (type != "synthesize") {
228+
throw std::invalid_argument("unknown request type: " + type);
229+
}
230+
if (!request.contains("id") || !request["id"].is_number_unsigned() ||
231+
request["id"].get<uint64_t>() == 0) {
232+
throw std::invalid_argument("id must be a positive integer");
233+
}
234+
id = request["id"].get<uint64_t>();
235+
static const std::set<std::string> expected{
236+
"type", "id", "text", "output"};
237+
std::set<std::string> actual;
238+
for (auto item = request.begin(); item != request.end(); ++item) {
239+
actual.insert(item.key());
240+
}
241+
if (actual != expected) {
242+
throw std::invalid_argument("synthesize request has unexpected fields");
243+
}
244+
if (!request["text"].is_string() ||
245+
request["text"].get_ref<const std::string&>().empty()) {
246+
throw std::invalid_argument("text must be a nonempty string");
247+
}
248+
if (!request["output"].is_string() ||
249+
request["output"].get_ref<const std::string&>().empty()) {
250+
throw std::invalid_argument("output must be a nonempty string");
251+
}
252+
if (*id <= last_request_id) {
253+
throw std::invalid_argument(
254+
"id must increase monotonically above " +
255+
std::to_string(last_request_id));
256+
}
257+
last_request_id = *id;
258+
259+
const std::filesystem::path output = request["output"].get<std::string>();
260+
if (const auto error = validate_output_path(output)) {
261+
throw std::invalid_argument(*error);
262+
}
263+
supertonic::SynthesisOptions options = defaults;
264+
options.text = request["text"].get<std::string>();
265+
const auto result = runner.synthesize(options);
266+
write_wav_without_overwrite(
267+
output,
268+
result.waveform,
269+
static_cast<int>(runner.metadata().sample_rate));
270+
const double audio_seconds = static_cast<double>(result.waveform.size()) /
271+
runner.metadata().sample_rate;
272+
write_json_line({
273+
{"type", "result"},
274+
{"id", *id},
275+
{"output", output.string()},
276+
{"samples", result.waveform.size()},
277+
{"audio_seconds", audio_seconds},
278+
{"synthesis_seconds", result.elapsed_seconds},
279+
{"rtf",
280+
audio_seconds > 0.0 ? result.elapsed_seconds / audio_seconds : 0.0},
281+
});
282+
} catch (const Json::parse_error&) {
283+
write_error(std::nullopt, "malformed JSON request");
284+
} catch (const std::exception& error) {
285+
std::cerr << "Supertonic JSONL request failed: " << error.what() << '\n';
286+
write_error(id, error.what());
287+
}
288+
}
289+
}
290+
45291
} // namespace
46292

47293
int main(int argc, char** argv) {
@@ -50,9 +296,14 @@ int main(int argc, char** argv) {
50296
gflags::ParseCommandLineFlags(&argc, &argv, true);
51297
try {
52298
if (FLAGS_pte.empty() || FLAGS_asset_dir.empty() ||
53-
FLAGS_voice_style.empty() || FLAGS_text.empty()) {
299+
FLAGS_voice_style.empty() ||
300+
(!FLAGS_server_jsonl && FLAGS_text.empty())) {
54301
throw std::invalid_argument(
55-
"--pte, --asset_dir, --voice_style, and --text are required");
302+
"--pte, --asset_dir, and --voice_style are required; --text is "
303+
"required unless --server_jsonl is set");
304+
}
305+
if (FLAGS_server_jsonl && FLAGS_warmup_text.empty()) {
306+
throw std::invalid_argument("--warmup_text must not be empty");
56307
}
57308
const std::filesystem::path pte(FLAGS_pte);
58309
const std::filesystem::path indexer =
@@ -70,14 +321,20 @@ int main(int argc, char** argv) {
70321
}
71322
require_file(style, "--voice_style");
72323

324+
const auto load_started = std::chrono::steady_clock::now();
73325
supertonic::SupertonicRunner runner(pte.string(), indexer.string());
326+
const double load_seconds = elapsed_seconds(load_started);
74327

75328
supertonic::SynthesisOptions options;
76329
options.text = FLAGS_text;
77330
options.language = FLAGS_language;
78331
options.voice_style_paths = {style};
79332
options.speed = static_cast<float>(FLAGS_speed);
80333
options.seed = FLAGS_seed;
334+
if (FLAGS_server_jsonl) {
335+
return run_jsonl_server(runner, options, load_seconds);
336+
}
337+
81338
auto result = runner.synthesize(options);
82339
if (!supertonic::write_pcm16_wav(
83340
FLAGS_output,

0 commit comments

Comments
 (0)