// jaylene slide // 0x04

.buckconfig

@@ -6,7 +6,9 @@
 [cells]
 root = .
 prelude = nix/build/prelude
-toolchains = toolchains
+straylight_prelude = prelude
+aleph = nix/build/aleph
+toolchains = nix/build/toolchains
 none = none
 
 [cell_aliases]
@@ -25,4 +27,4 @@ materializations = deferred
 digest_algorithms = BLAKE3
 
 [build]
-execution_platforms = toolchains//:default
+execution_platforms = toolchains//:local

.clang-format

@@ -1 +1 @@
-/nix/store/a8npkxz9kfij786yqaay5fbib4gdsbv1-source/nix/configs/.clang-format
+/nix/store/dm3zx6x2cfaccnkg75ndgpjnhz4sz2ak-source/nix/configs/.clang-format

.clang-tidy

@@ -1 +1 @@
-/nix/store/a8npkxz9kfij786yqaay5fbib4gdsbv1-source/nix/configs/.clang-tidy
+/nix/store/dm3zx6x2cfaccnkg75ndgpjnhz4sz2ak-source/nix/configs/.clang-tidy

.clangd

@@ -1 +1 @@
-/nix/store/a8npkxz9kfij786yqaay5fbib4gdsbv1-source/nix/configs/.clangd
+/nix/store/dm3zx6x2cfaccnkg75ndgpjnhz4sz2ak-source/nix/configs/.clangd

.envrc

@@ -0,0 +1 @@
+use flake

.gitignore

@@ -3,3 +3,5 @@ buck-out/
 .buckconfig.local
 compile_commands.json
 result
+.direnv
+nix/build

.rustfmt.toml

@@ -1 +1 @@
-/nix/store/a8npkxz9kfij786yqaay5fbib4gdsbv1-source/nix/configs/.rustfmt.toml
+/nix/store/dm3zx6x2cfaccnkg75ndgpjnhz4sz2ak-source/nix/configs/.rustfmt.toml

.stylua.toml

@@ -1 +1 @@
-/nix/store/a8npkxz9kfij786yqaay5fbib4gdsbv1-source/nix/configs/.stylua.toml
+/nix/store/dm3zx6x2cfaccnkg75ndgpjnhz4sz2ak-source/nix/configs/.stylua.toml

BUCK

@@ -4,8 +4,13 @@
 # Library paths for tokenizers-cpp come from .buckconfig.local [slide] section
 # which is auto-generated by `nix develop`
 # The haskell_ffi_binary rule reads these paths via read_root_config()
+#
+# Benchmarks: buck2 run //:bench
+# Run specific: buck2 run //:bench -- encode decode
+# Throughput burst: buck2 run //:bench -- throughput
+# Markov SSE: buck2 run //:markov
 
-load("@toolchains//:haskell.bzl", "haskell_ffi_binary")
+load("@toolchains//:haskell.bzl", "haskell_binary", "haskell_ffi_binary", "haskell_ffi_test")
 haskell_ffi_binary(
     name = "slide",
     hs_srcs = ["src/Slide/Chunk.hs", "src/Slide/Configuration.hs", "src/Slide/HotTable.hs", "src/Slide/Model.hs", "src/Slide/Parse.hs", "src/Slide/Provider.hs", "src/Slide/Provider/HTTP2.hs", "src/Slide/Provider/OpenAI.hs", "src/Slide/Provider/OpenRouter.hs", "src/Slide/Provider/Vertex/Anthropic.hs", "src/Slide/Tokenizer.hs", "src/Slide/Tokenizer/FFI.hs", "src/Slide/Wire/Decode.hs", "src/Slide/Wire/Encode.hs", "src/Slide/Wire/Frame.hs", "src/Slide/Wire/Types.hs", "src/Slide/Wire/Varint.hs", "app/Main.hs"],
@@ -17,3 +22,39 @@ haskell_ffi_binary(
     include_dirs = ["cbits"],
     visibility = ["PUBLIC"],
 )
+
+haskell_ffi_test(
+    name = "slide-test",
+    hs_srcs = ["src/Slide/Chunk.hs", "src/Slide/Configuration.hs", "src/Slide/HotTable.hs", "src/Slide/Model.hs", "src/Slide/Parse.hs", "src/Slide/Provider.hs", "src/Slide/Provider/HTTP2.hs", "src/Slide/Provider/OpenAI.hs", "src/Slide/Provider/OpenRouter.hs", "src/Slide/Provider/Vertex/Anthropic.hs", "src/Slide/Tokenizer.hs", "src/Slide/Tokenizer/FFI.hs", "src/Slide/Wire/Decode.hs", "src/Slide/Wire/Encode.hs", "src/Slide/Wire/Frame.hs", "src/Slide/Wire/Types.hs", "src/Slide/Wire/Varint.hs", "test/Main.hs", "test/ChunkSpec.hs", "test/ConfigurationSpec.hs", "test/DecodeSpec.hs", "test/EncodeSpec.hs", "test/FrameSpec.hs", "test/HotTableSpec.hs", "test/ModelSpec.hs", "test/ParseSpec.hs", "test/RoundtripSpec.hs", "test/StressSpec.hs", "test/TokenizerFFISpec.hs", "test/ToolCallSpec.hs", "test/TypesSpec.hs", "test/VarintSpec.hs"],
+    cxx_srcs = ["cbits/tokenizers_c.cpp"],
+    packages = ["base", "aeson", "async", "blake3", "bytestring", "case-insensitive", "containers", "crypton", "data-default-class", "dhall", "http2", "http-semantics", "http-types", "katip", "megaparsec", "memory", "network", "optparse-applicative", "prometheus-client", "prometheus-metrics-ghc", "random", "text", "time", "time-manager", "tls", "vector", "wai", "warp", "zeromq4-haskell", "hspec", "QuickCheck", "temporary"],
+    language_extensions = ["BangPatterns", "CApiFFI", "DerivingStrategies", "ForeignFunctionInterface", "LambdaCase", "NumericUnderscores", "OverloadedStrings", "PatternSynonyms", "StrictData", "PackageImports", "ScopedTypeVariables"],
+    ghc_options = ["-O0", "-threaded", "-rtsopts", "-with-rtsopts=-N", "-isrc", "-itest"],
+    extra_libs = ["tokenizers_cpp", "tokenizers_c", "sentencepiece"],
+    include_dirs = ["cbits"],
+    visibility = ["PUBLIC"],
+)
+
+haskell_ffi_binary(
+    name = "bench",
+    hs_srcs = ["src/Slide/Chunk.hs", "src/Slide/Configuration.hs", "src/Slide/HotTable.hs", "src/Slide/Model.hs", "src/Slide/Parse.hs", "src/Slide/Provider.hs", "src/Slide/Provider/HTTP2.hs", "src/Slide/Provider/OpenAI.hs", "src/Slide/Provider/OpenRouter.hs", "src/Slide/Provider/Vertex/Anthropic.hs", "src/Slide/Tokenizer.hs", "src/Slide/Tokenizer/FFI.hs", "src/Slide/Wire/Decode.hs", "src/Slide/Wire/Encode.hs", "src/Slide/Wire/Frame.hs", "src/Slide/Wire/Types.hs", "src/Slide/Wire/Varint.hs", "bench/Main.hs"],
+    cxx_srcs = ["cbits/tokenizers_c.cpp"],
+    packages = ["base", "aeson", "async", "blake3", "bytestring", "case-insensitive", "containers", "crypton", "data-default-class", "dhall", "http2", "http-semantics", "http-types", "katip", "megaparsec", "memory", "network", "optparse-applicative", "prometheus-client", "prometheus-metrics-ghc", "random", "text", "time", "time-manager", "tls", "vector", "wai", "warp", "zeromq4-haskell", "clock", "deepseq"],
+    language_extensions = ["BangPatterns", "CApiFFI", "DerivingStrategies", "ForeignFunctionInterface", "LambdaCase", "NumericUnderscores", "OverloadedStrings", "PatternSynonyms", "StrictData", "PackageImports"],
+    ghc_options = ["-O2", "-threaded", "-rtsopts", "-with-rtsopts=-N -A64m -I0", "-isrc"],
+    extra_libs = ["tokenizers_cpp", "tokenizers_c", "sentencepiece"],
+    include_dirs = ["cbits"],
+    visibility = ["PUBLIC"],
+)
+
+haskell_ffi_binary(
+    name = "markov",
+    hs_srcs = ["test/MarkovSSE.hs", "src/Slide/HotTable.hs", "src/Slide/Tokenizer.hs", "src/Slide/Tokenizer/FFI.hs", "src/Slide/Wire/Frame.hs", "src/Slide/Wire/Types.hs", "src/Slide/Wire/Varint.hs"],
+    cxx_srcs = ["cbits/tokenizers_c.cpp"],
+    packages = ["base", "aeson", "bytestring", "containers", "optparse-applicative", "random", "text", "vector", "zeromq4-haskell"],
+    language_extensions = ["BangPatterns", "CApiFFI", "DerivingStrategies", "ForeignFunctionInterface", "LambdaCase", "NumericUnderscores", "OverloadedStrings", "PatternSynonyms", "StrictData", "PackageImports"],
+    ghc_options = ["-O2", "-threaded", "-rtsopts", "-with-rtsopts=-N", "-isrc", "-main-is", "MarkovSSE"],
+    extra_libs = ["tokenizers_cpp", "tokenizers_c", "sentencepiece"],
+    include_dirs = ["cbits"],
+    visibility = ["PUBLIC"],
+)

BUILD.dhall

@@ -98,6 +98,41 @@ let extraLibs =
       , "sentencepiece"
       ]
 
+-- Benchmark sources
+let benchSrcs = [ "bench/Main.hs" ]
+
+-- Benchmark packages (library packages + bench-specific)
+let benchPackages = packages #
+      [ "clock"
+      , "deepseq"
+      ]
+
+-- Test sources
+let testSrcs =
+      [ "test/Main.hs"
+      , "test/ChunkSpec.hs"
+      , "test/ConfigurationSpec.hs"
+      , "test/DecodeSpec.hs"
+      , "test/EncodeSpec.hs"
+      , "test/FrameSpec.hs"
+      , "test/HotTableSpec.hs"
+      , "test/ModelSpec.hs"
+      , "test/ParseSpec.hs"
+      , "test/RoundtripSpec.hs"
+      , "test/StressSpec.hs"
+      , "test/TokenizerFFISpec.hs"
+      , "test/ToolCallSpec.hs"
+      , "test/TypesSpec.hs"
+      , "test/VarintSpec.hs"
+      ]
+
+-- Test packages (library packages + test-specific)
+let testPackages = packages #
+      [ "hspec"
+      , "QuickCheck"
+      , "temporary"
+      ]
+
 -- slide executable with FFI
 let slide =
       (A.haskellFFIBinary "slide" (librarySrcs # appSrcs) cxxSrcs)
@@ -107,14 +142,69 @@ let slide =
         with extra_libs = extraLibs
         with include_dirs = [ "cbits" ]
 
-in  { rules = [ S.haskellFFIBinary slide ]
-    , header = ''
+-- test executable with FFI
+let slideTest =
+      (A.haskellFFIBinary "slide-test" (librarySrcs # testSrcs) cxxSrcs)
+        with packages = testPackages
+        with language_extensions = extensions # [ "ScopedTypeVariables" ]
+        with ghc_options = [ "-O0", "-threaded", "-rtsopts", "-with-rtsopts=-N", "-isrc", "-itest" ]
+        with extra_libs = extraLibs
+        with include_dirs = [ "cbits" ]
+
+-- benchmark executable with FFI (optimized)
+let slideBench =
+      (A.haskellFFIBinary "bench" (librarySrcs # benchSrcs) cxxSrcs)
+        with packages = benchPackages
+        with language_extensions = extensions
+        with ghc_options = [ "-O2", "-threaded", "-rtsopts", "-with-rtsopts=-N -A64m -I0", "-isrc" ]
+        with extra_libs = extraLibs
+        with include_dirs = [ "cbits" ]
+
+-- markov SSE generator (needs FFI for tokenizer)
+let markovSrcs =
+      [ "test/MarkovSSE.hs"
+      , "src/Slide/HotTable.hs"
+      , "src/Slide/Tokenizer.hs"
+      , "src/Slide/Tokenizer/FFI.hs"
+      , "src/Slide/Wire/Frame.hs"
+      , "src/Slide/Wire/Types.hs"
+      , "src/Slide/Wire/Varint.hs"
+      ]
+
+let markovPackages =
+      [ "base"
+      , "aeson"
+      , "bytestring"
+      , "containers"
+      , "optparse-applicative"
+      , "random"
+      , "text"
+      , "vector"
+      , "zeromq4-haskell"
+      ]
+
+let markov =
+      (A.haskellFFIBinary "markov" markovSrcs cxxSrcs)
+        with packages = markovPackages
+        with language_extensions = extensions
+        with ghc_options = [ "-O2", "-threaded", "-rtsopts", "-with-rtsopts=-N", "-isrc", "-main-is", "MarkovSSE" ]
+        with extra_libs = extraLibs
+        with include_dirs = [ "cbits" ]
+
+in  { rules = [ S.haskellFFIBinary slide, S.haskellFFITest slideTest, S.haskellFFIBinary slideBench, S.haskellFFIBinary markov ]
+    , header =
+        ''
         # Generated from BUILD.dhall
         # 
         # Library paths for tokenizers-cpp come from .buckconfig.local [slide] section
         # which is auto-generated by `nix develop`
         # The haskell_ffi_binary rule reads these paths via read_root_config()
+        #
+        # Benchmarks: buck2 run //:bench
+        # Run specific: buck2 run //:bench -- encode decode
+        # Throughput burst: buck2 run //:bench -- throughput
+        # Markov SSE: buck2 run //:markov
 
-        load("@toolchains//:haskell.bzl", "haskell_ffi_binary")
+        load("@toolchains//:haskell.bzl", "haskell_binary", "haskell_ffi_binary", "haskell_ffi_test")
         ''
     }

README.md

@@ -23,7 +23,7 @@ Examples:
 
 The `0x0N` suffix is a sequential counter within the branch for easy reference.
 
----
+______________________________________________________________________
 
 ```
   ╷┌─┐┐ ┬┬  ┌─┐┌┐┐┌─┐  ┐─┐┬  o┬─┐┌─┐
@@ -46,10 +46,11 @@ behind her “fucks with me. You got that?”
 **jaylene-slide** is a console cowboy that jacks into OpenAI-compatible inference endpoints (Baseten, Together, Fireworks, etc.), parses their 650-byte-per-token SSE/JSON garbage, and emits clean SIGIL binary frames over ZMQ.
 
 It supports:
-*   **Semantic Chunking**: Splits streams intelligently on token boundaries, newlines, and special tags.
-*   **Chain of Thought**: Natively handles reasoning models (DeepSeek R1, Kimi K2.5) with optional visualization.
-*   **High Performance**: Uses HTTP/2 (via `http2-client`), optimized Tokenizer FFI (HuggingFace via `tokenizers-cpp`), and Zero-Copy ZMQ patterns.
-*   **Structured Logging**: Observability via Katip (JSON or colored terminal output).
+
+- **Semantic Chunking**: Splits streams intelligently on token boundaries, newlines, and special tags.
+- **Chain of Thought**: Natively handles reasoning models (DeepSeek R1, Kimi K2.5) with optional visualization.
+- **High Performance**: Uses HTTP/2 (via `http2-client`), optimized Tokenizer FFI (HuggingFace via `tokenizers-cpp`), and Zero-Copy ZMQ patterns.
+- **Structured Logging**: Observability via Katip (JSON or colored terminal output).
 
 ## Why
 
@@ -101,8 +102,8 @@ One byte. Because `" const"` is hot token #42.
 
 ### Prerequisites
 
-*   Nix (with flakes enabled)
-*   OR Cabal + GHC 9.12 + `libtokenizers`
+- Nix (with flakes enabled)
+- OR Cabal + GHC 9.12 + `libtokenizers`
 
 ### Running the Ingress (Jack Mode)
 
@@ -183,7 +184,8 @@ This invokes `fourmolu`, `cabal-fmt`, and `nixpkgs-fmt`.
 Some models (like Kimi K2.5 or OpenAI models) distribute `tiktoken.model` or raw vocab files.
 
 **Workaround**: Use a compatible `tokenizer.json` from a related model.
-*   **Kimi K2.5**: Use **Qwen 2.5** tokenizer (`Qwen/Qwen2.5-7B-Instruct`). They share the same vocabulary structure.
+
+- **Kimi K2.5**: Use **Qwen 2.5** tokenizer (`Qwen/Qwen2.5-7B-Instruct`). They share the same vocabulary structure.
 
 ### HTTP/2 Notes
 
@@ -192,28 +194,31 @@ This project strictly requires `http2-client` (which depends on `http2` < 5.0) t
 ## Logging & Observability
 
 ### Logging (Katip)
-*   **Console**: Human-readable, colored logs on stderr.
-*   **JSON**: Use `--json-logs` for machine-readable output (Datadog/CloudWatch friendly).
-*   **Verbose**: Use `-v` or `--verbose` to see individual chunk arrivals ("flying messages").
-*   **Context**: All logs include `slide_id` (process) and `http_id` (transaction) correlation IDs.
+
+- **Console**: Human-readable, colored logs on stderr.
+- **JSON**: Use `--json-logs` for machine-readable output (Datadog/CloudWatch friendly).
+- **Verbose**: Use `-v` or `--verbose` to see individual chunk arrivals ("flying messages").
+- **Context**: All logs include `slide_id` (process) and `http_id` (transaction) correlation IDs.
 
 ### Metrics (Prometheus)
+
 Exposes metrics on a dedicated port (default 9090, configurable via `--metrics-port`).
 
 Endpoint: `GET /` or `GET /metrics`
 
-*   `slide_frames_emitted_total`: Counter
-*   `slide_bytes_emitted_total`: Counter
-*   `slide_tokens_processed_total`: Counter
-*   GHC Runtime Metrics (GC, Heap, Threads)
+- `slide_frames_emitted_total`: Counter
+- `slide_bytes_emitted_total`: Counter
+- `slide_tokens_processed_total`: Counter
+- GHC Runtime Metrics (GC, Heap, Threads)
 
 ## Style Guide
 
 We follow the **Straylight Production Haskell** conventions:
-*   Optimize for disambiguation (explicit types, full variable names).
-*   Flat control flow (guards over nesting).
-*   Strict warnings (`-Wall -Werror`).
-*   Unicode delimiters (`// typographical // conventions`).
+
+- Optimize for disambiguation (explicit types, full variable names).
+- Flat control flow (guards over nesting).
+- Strict warnings (`-Wall -Werror`).
+- Unicode delimiters (`// typographical // conventions`).
 
 See `STYLE.md` (if it existed) or the codebase itself for examples.